diff --git a/frontend/app_flowy/packages/flowy_editor/lib/service/selection_service.dart b/frontend/app_flowy/packages/flowy_editor/lib/service/selection_service.dart
index 1786eca05f..9ae0082001 100644
--- a/frontend/app_flowy/packages/flowy_editor/lib/service/selection_service.dart
+++ b/frontend/app_flowy/packages/flowy_editor/lib/service/selection_service.dart
@@ -339,9 +339,10 @@ class _FlowySelectionState extends State<FlowySelection>
           start: isDownward ? start : end, end: isDownward ? end : start);
       debugPrint('[_onPanUpdate] isDownward = $isDownward, $selection');
       editorState.updateCursorSelection(selection);
+
+      _scrollUpOrDownIfNeeded(panEndOffset!, isDownward);
     }
 
-    _scrollUpOrDownIfNeeded(panEndOffset!);
     _showDebugLayerIfNeeded();
   }
 
@@ -466,7 +467,7 @@ class _FlowySelectionState extends State<FlowySelection>
     return NodeIterator(stateTree, startNode, endNode).toList();
   }
 
-  void _scrollUpOrDownIfNeeded(Offset offset) {
+  void _scrollUpOrDownIfNeeded(Offset offset, bool isDownward) {
     final dy = editorState.service.scrollService?.dy;
     if (dy == null) {
       assert(false, 'Dy could not be null');
@@ -478,10 +479,10 @@ class _FlowySelectionState extends State<FlowySelection>
     /// TODO: It is necessary to calculate the relative speed
     ///   according to the gap and move forward more gently.
     final distance = 10.0;
-    if (offset.dy <= topLimit) {
+    if (offset.dy <= topLimit && !isDownward) {
       // up
       editorState.service.scrollService?.scrollTo(dy - distance);
-    } else if (offset.dy >= bottomLimit) {
+    } else if (offset.dy >= bottomLimit && isDownward) {
       //down
       editorState.service.scrollService?.scrollTo(dy + distance);
     }