Fix sync loop for too long path

mergin/client.py

@@ -2,7 +2,6 @@
 import math
 import os
 import json
-import shutil
 import zlib
 import base64
 import urllib.parse
@@ -68,6 +67,7 @@
     is_version_acceptable,
     normalize_role,
 )
+from . import fs
 from .version import __version__
 
 try:
@@ -1246,7 +1246,7 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi
             if len(diffs) > 1:
                 mp.geodiff.concat_changes(diffs, output_diff)
             elif len(diffs) == 1:
-                shutil.copy(diffs[0], output_diff)
+                fs.copy(diffs[0], output_diff)
 
     def download_file_diffs(self, project_dir, file_path, versions):
         """Download file diffs for specified versions if they are not present
@@ -1377,7 +1377,7 @@ def reset_local_changes(self, directory: str, files_to_reset: typing.List[str] =
         # remove all added files
         for file in push_changes["added"]:
             if all_files or file["path"] in files_to_reset:
-                os.remove(mp.fpath(file["path"]))
+                fs.remove(mp.fpath(file["path"]))
 
         # update files get override with previous version
         for file in push_changes["updated"]:

mergin/client_pull.py

@@ -26,6 +26,7 @@
 from .models import ProjectDelta, ProjectDeltaChange, PullAction
 from .merginproject import MerginProject
 from .utils import cleanup_tmp_dir, save_to_file
+from . import fs
 from typing import List, Optional
 
 # status = download_project_async(...)
@@ -142,7 +143,7 @@ def download_blocking(self, mc, mp):
         if resp.status in [200, 206]:
             mp.log.debug(f"Download finished: {self.diff_id}")
             save_to_file(resp, self.download_file_path)
-            self.size = os.path.getsize(self.download_file_path)
+            self.size = fs.getsize(self.download_file_path)
         else:
             mp.log.error(f"Download failed: {self.diff_id}")
             raise ClientError(f"Failed to download of diff file {self.diff_id} to {self.download_file_path}")
@@ -164,19 +165,19 @@ def __init__(self, dest_file, downloaded_items: typing.List[DownloadQueueItem],
     def from_chunks(self):
         """Merges downloaded chunks into a single file at dest_file path"""
         file_dir = os.path.dirname(self.dest_file)
-        os.makedirs(file_dir, exist_ok=True)
+        fs.makedirs(file_dir, exist_ok=True)
 
-        with open(self.dest_file, "wb") as final:
+        with fs.open_file(self.dest_file, "wb") as final:
             for item in self.downloaded_items:
-                with open(item.download_file_path, "rb") as chunk:
+                with fs.open_file(item.download_file_path, "rb") as chunk:
                     shutil.copyfileobj(chunk, final)
-                os.remove(item.download_file_path)
+                fs.remove(item.download_file_path)
 
         if not self.size_check:
             return
         expected_size = sum(item.size for item in self.downloaded_items)
-        if os.path.getsize(self.dest_file) != expected_size:
-            os.remove(self.dest_file)
+        if fs.getsize(self.dest_file) != expected_size:
+            fs.remove(self.dest_file)
             raise ClientError("Download of file {} failed. Please try it again.".format(self.dest_file))
 
 
@@ -555,7 +556,7 @@ def pull_project_async(mc, directory) -> Optional[PullJob]:
             pull_action_type == PullActionType.COPY_CONFLICT and change.type == DeltaChangeType.UPDATE_DIFF
         ):
             basefile = mp.fpath_meta(change.path)
-            if not os.path.exists(basefile):
+            if not fs.exists(basefile):
                 # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file
                 # or we removed it within previous pull because we failed to apply patch the older version for some reason).
                 # But it's not a problem - we will download the newest version and we're sorted.
@@ -722,7 +723,7 @@ def pull_project_finalize(job: PullJob):
         basefile = job.mp.fpath_meta(file_path)
         server_file = job.mp.fpath(file_path, job.tmp_dir.name)
 
-        shutil.copy(basefile, server_file)
+        fs.copy(basefile, server_file)
         diffs = [job.mp.fpath(f, job.tmp_dir.name) for f in file_diffs]
         patch_error = job.mp.apply_diffs(server_file, diffs)
         if patch_error:
@@ -735,7 +736,7 @@ def pull_project_finalize(job: PullJob):
             job.mp.log.error("Diffs we were applying: " + str(diffs))
             job.mp.log.error("Removing basefile because it would be corrupted anyway...")
             job.mp.log.info("--- pull aborted")
-            os.remove(basefile)
+            fs.remove(basefile)
             raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile))
     conflicts = []
     job.mp.log.info(f"--- applying pull actions {job.pull_actions}")
@@ -830,7 +831,7 @@ def download_diffs_async(mc, project_directory, file_path, versions):
             diff_only=True,
         )
         dest_file_path = mp.fpath_cache(diff["path"], version=file["version"])
-        if os.path.exists(dest_file_path):
+        if fs.exists(dest_file_path):
             continue
         download_files.append(DownloadFile(dest_file_path, items))
         download_list.extend(items)

mergin/client_push.py

@@ -18,7 +18,6 @@
 import pprint
 import tempfile
 import concurrent.futures
-import os
 import time
 from typing import List, Tuple, Optional, ByteString
 
@@ -35,6 +34,7 @@
 from .merginproject import MerginProject, pygeodiff
 from .editor import filter_changes
 from .utils import get_data_checksum, cleanup_tmp_dir
+from . import fs
 
 POST_JSON_HEADERS = {"Content-Type": "application/json"}
 
@@ -114,7 +114,7 @@ def upload_chunk_v2_api(self, data: ByteString, checksum: str):
         self.mc.upload_chunks_cache.add(checksum, self.server_chunk_id)
 
     def upload_blocking(self):
-        with open(self.file_path, "rb") as file_handle:
+        with fs.open_file(self.file_path, "rb") as file_handle:
             file_handle.seek(self.chunk_index * UPLOAD_CHUNK_SIZE)
             data = file_handle.read(UPLOAD_CHUNK_SIZE)
             checksum_str = get_data_checksum(data)
@@ -508,8 +508,8 @@ def remove_diff_files(job: UploadJob) -> None:
         diff = change.get_diff()
         if diff:
             diff_file = job.mp.fpath_meta(diff.path)
-            if os.path.exists(diff_file):
-                os.remove(diff_file)
+            if fs.exists(diff_file):
+                fs.remove(diff_file)
 
 
 def get_push_changes_batch(mc, directory: str) -> Tuple[LocalProjectChanges, int]:

mergin/fs.py

@@ -0,0 +1,51 @@
+"""
+Thin wrappers around the standard-library filesystem calls used by the sync code.
+
+Every wrapper applies utils.long_path() to its path argument(s) so that Windows paths
+longer than MAX_PATH are handled transparently.
+
+Use these instead of calling os.* / shutil.* / open() / sqlite3.connect() directly on
+project file paths.
+"""
+
+import os
+import shutil
+import sqlite3
+
+from .utils import long_path
+
+
+def remove(path):
+    os.remove(long_path(path))
+
+
+def exists(path) -> bool:
+    return os.path.exists(long_path(path))
+
+
+def getsize(path) -> int:
+    return os.path.getsize(long_path(path))
+
+
+def getmtime(path) -> float:
+    return os.path.getmtime(long_path(path))
+
+
+def copy(src, dst):
+    return shutil.copy(long_path(src), long_path(dst))
+
+
+def walk(path, **kwargs):
+    return os.walk(long_path(path), **kwargs)
+
+
+def makedirs(path, exist_ok=False):
+    os.makedirs(long_path(path), exist_ok=exist_ok)
+
+
+def connect(path):
+    return sqlite3.connect(long_path(path))
+
+
+def open_file(path, *args, **kwargs):
+    return open(long_path(path), *args, **kwargs)