From 2c948d121e738c79fed027a06c6f2c044cd52776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=AB=E6=8C=AF=E5=AE=87?= Date: Mon, 15 Jan 2024 16:04:47 +0800 Subject: [PATCH 1/7] get all releases --- github-release.py | 122 ++++++++++++++++++++++++++++------------------ 1 file changed, 75 insertions(+), 47 deletions(-) diff --git a/github-release.py b/github-release.py index 59d79c5..f9af136 100755 --- a/github-release.py +++ b/github-release.py @@ -21,20 +21,21 @@ TIMEOUT_OPTION = (7, 10) total_size = 0 -def sizeof_fmt(num, suffix='iB'): - for unit in ['','K','M','G','T','P','E','Z']: + +def sizeof_fmt(num, suffix="iB"): + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: if abs(num) < 1024.0: return "%3.2f%s%s" % (num, unit, suffix) num /= 1024.0 - return "%.2f%s%s" % (num, 'Y', suffix) + return "%.2f%s%s" % (num, "Y", suffix) + # wrap around requests.get to use token if available def github_get(*args, **kwargs): - headers = kwargs['headers'] if 'headers' in kwargs else {} - if 'GITHUB_TOKEN' in os.environ: - headers['Authorization'] = 'token {}'.format( - os.environ['GITHUB_TOKEN']) - kwargs['headers'] = headers + headers = kwargs["headers"] if "headers" in kwargs else {} + if "GITHUB_TOKEN" in os.environ: + headers["Authorization"] = "token {}".format(os.environ["GITHUB_TOKEN"]) + kwargs["headers"] = headers return requests.get(*args, **kwargs) @@ -44,7 +45,12 @@ def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size: r.raise_for_status() tmp_dst_file = None try: - with tempfile.NamedTemporaryFile(prefix="." + dst_file.name + ".", suffix=".tmp", dir=dst_file.parent, delete=False) as f: + with tempfile.NamedTemporaryFile( + prefix="." + dst_file.name + ".", + suffix=".tmp", + dir=dst_file.parent, + delete=False, + ) as f: tmp_dst_file = Path(f.name) for chunk in r.iter_content(chunk_size=1024**2): if chunk: # filter out keep-alive new chunks @@ -53,7 +59,9 @@ def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size: # check for downloaded size downloaded_size = tmp_dst_file.stat().st_size if remote_size != -1 and downloaded_size != remote_size: - raise Exception(f'File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes') + raise Exception( + f"File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes" + ) os.utime(tmp_dst_file, (remote_ts, remote_ts)) tmp_dst_file.chmod(0o644) tmp_dst_file.replace(dst_file) @@ -71,8 +79,7 @@ def downloading_worker(q): url, dst_file, working_dir, updated, remote_size = item - print("downloading", url, "to", - dst_file.relative_to(working_dir), flush=True) + print("downloading", url, "to", dst_file.relative_to(working_dir), flush=True) try: do_download(url, dst_file, updated, remote_size) except Exception: @@ -86,30 +93,35 @@ def downloading_worker(q): def create_workers(n): task_queue = queue.Queue() for i in range(n): - t = threading.Thread(target=downloading_worker, args=(task_queue, )) + t = threading.Thread(target=downloading_worker, args=(task_queue,)) t.start() return task_queue def ensure_safe_name(filename): - filename = filename.replace('\0', ' ') - if filename == '.': - return ' .' - elif filename == '..': - return '. .' + filename = filename.replace("\0", " ") + if filename == ".": + return " ." + elif filename == "..": + return ". ." else: - return filename.replace('/', '\\').replace('\\', '_') + return filename.replace("/", "\\").replace("\\", "_") def main(): import argparse + parser = argparse.ArgumentParser() parser.add_argument("--base-url", default=BASE_URL) parser.add_argument("--working-dir", default=WORKING_DIR) - parser.add_argument("--workers", default=1, type=int, - help='number of concurrent downloading jobs') - parser.add_argument("--fast-skip", action='store_true', - help='do not verify size and timestamp of existing files') + parser.add_argument( + "--workers", default=1, type=int, help="number of concurrent downloading jobs" + ) + parser.add_argument( + "--fast-skip", + action="store_true", + help="do not verify size and timestamp of existing files", + ) parser.add_argument("--config", default=CONFIG) args = parser.parse_args() @@ -124,14 +136,15 @@ def main(): with open(args.config, "r") as f: REPOS = json.load(f) - def download(release, release_dir, tarball = False): + def download(release, release_dir, tarball=False): global total_size if tarball: - url = release['tarball_url'] + url = release["tarball_url"] updated = datetime.strptime( - release['published_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp() - dst_file = release_dir / 'repo-snapshot.tar.gz' + release["published_at"], "%Y-%m-%dT%H:%M:%SZ" + ).timestamp() + dst_file = release_dir / "repo-snapshot.tar.gz" remote_filelist.append(dst_file.relative_to(working_dir)) if dst_file.is_file(): @@ -141,19 +154,21 @@ def download(release, release_dir, tarball = False): # tarball has no size information, use -1 to skip size check task_queue.put((url, dst_file, working_dir, updated, -1)) - for asset in release['assets']: - url = asset['browser_download_url'] + for asset in release["assets"]: + url = asset["browser_download_url"] updated = datetime.strptime( - asset['updated_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp() - dst_file = release_dir / ensure_safe_name(asset['name']) + asset["updated_at"], "%Y-%m-%dT%H:%M:%SZ" + ).timestamp() + dst_file = release_dir / ensure_safe_name(asset["name"]) remote_filelist.append(dst_file.relative_to(working_dir)) - remote_size = asset['size'] + remote_size = asset["size"] total_size += remote_size if dst_file.is_file(): if args.fast_skip: - print("fast skipping", dst_file.relative_to( - working_dir), flush=True) + print( + "fast skipping", dst_file.relative_to(working_dir), flush=True + ) continue else: stat = dst_file.stat() @@ -161,10 +176,12 @@ def download(release, release_dir, tarball = False): local_mtime = stat.st_mtime # print(f"{local_filesize} vs {asset['size']}") # print(f"{local_mtime} vs {updated}") - if local_mtime > updated or \ - remote_size == local_filesize and local_mtime == updated: - print("skipping", dst_file.relative_to( - working_dir), flush=True) + if ( + local_mtime > updated + or remote_size == local_filesize + and local_mtime == updated + ): + print("skipping", dst_file.relative_to(working_dir), flush=True) continue else: dst_file.parent.mkdir(parents=True, exist_ok=True) @@ -182,10 +199,10 @@ def link_latest(name, repo_dir): pass for cfg in REPOS: - flat = False # build a folder for each release - versions = 1 # keep only one release - tarball = False # do not download the tarball - prerelease = False # filter out pre-releases + flat = False # build a folder for each release + versions = 1 # keep only one release + tarball = False # do not download the tarball + prerelease = False # filter out pre-releases if isinstance(cfg, str): repo = cfg else: @@ -203,17 +220,27 @@ def link_latest(name, repo_dir): print(f"syncing {repo} to {repo_dir}") try: - r = github_get(f"{args.base_url}{repo}/releases") + headers = {"Accept": "application/vnd.github+json"} + r = github_get( + f"{args.base_url}{repo}/releases?per_page=100", headers=headers + ) r.raise_for_status() releases = r.json() + + next_url = r.headers["link"] + if next_url: + for url_str in next_url.split(","): + r = github_get(url_str.split(";")[0].strip()[1:-1], headers=headers) + r.raise_for_status() + releases.extend(r.json()) except: traceback.print_exc() break n_downloaded = 0 for release in releases: - if not release['draft'] and (prerelease or not release['prerelease']): - name = ensure_safe_name(release['name'] or release['tag_name']) + if not release["draft"] and (prerelease or not release["prerelease"]): + name = ensure_safe_name(release["name"] or release["tag_name"]) if len(name) == 0: print("Error: Unnamed release") continue @@ -238,7 +265,7 @@ def link_latest(name, repo_dir): if cleaning: local_filelist = [] - for local_file in working_dir.glob('**/*'): + for local_file in working_dir.glob("**/*"): if local_file.is_file(): local_filelist.append(local_file.relative_to(working_dir)) @@ -247,7 +274,7 @@ def link_latest(name, repo_dir): old_file = working_dir / old_file old_file.unlink() - for local_dir in working_dir.glob('*/*/*'): + for local_dir in working_dir.glob("*/*/*"): if local_dir.is_dir(): try: # remove empty dirs only @@ -257,6 +284,7 @@ def link_latest(name, repo_dir): print("Total size is", sizeof_fmt(total_size, suffix="")) + if __name__ == "__main__": main() From 7b1d7701c6b1ca8fd22d073d4940d26ca085badd Mon Sep 17 00:00:00 2001 From: jimorsm Date: Fri, 19 Jan 2024 15:31:13 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E6=9B=B4=E6=AD=A3link=E5=A4=84=E7=90=86?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- github-release.py | 135 ++++++++++++++++++++-------------------------- 1 file changed, 58 insertions(+), 77 deletions(-) diff --git a/github-release.py b/github-release.py index f9af136..1c74989 100755 --- a/github-release.py +++ b/github-release.py @@ -21,21 +21,20 @@ TIMEOUT_OPTION = (7, 10) total_size = 0 - -def sizeof_fmt(num, suffix="iB"): - for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: +def sizeof_fmt(num, suffix='iB'): + for unit in ['','K','M','G','T','P','E','Z']: if abs(num) < 1024.0: return "%3.2f%s%s" % (num, unit, suffix) num /= 1024.0 - return "%.2f%s%s" % (num, "Y", suffix) - + return "%.2f%s%s" % (num, 'Y', suffix) # wrap around requests.get to use token if available def github_get(*args, **kwargs): - headers = kwargs["headers"] if "headers" in kwargs else {} - if "GITHUB_TOKEN" in os.environ: - headers["Authorization"] = "token {}".format(os.environ["GITHUB_TOKEN"]) - kwargs["headers"] = headers + headers = kwargs['headers'] if 'headers' in kwargs else {} + if 'GITHUB_TOKEN' in os.environ: + headers['Authorization'] = 'token {}'.format( + os.environ['GITHUB_TOKEN']) + kwargs['headers'] = headers return requests.get(*args, **kwargs) @@ -45,12 +44,7 @@ def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size: r.raise_for_status() tmp_dst_file = None try: - with tempfile.NamedTemporaryFile( - prefix="." + dst_file.name + ".", - suffix=".tmp", - dir=dst_file.parent, - delete=False, - ) as f: + with tempfile.NamedTemporaryFile(prefix="." + dst_file.name + ".", suffix=".tmp", dir=dst_file.parent, delete=False) as f: tmp_dst_file = Path(f.name) for chunk in r.iter_content(chunk_size=1024**2): if chunk: # filter out keep-alive new chunks @@ -59,9 +53,7 @@ def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size: # check for downloaded size downloaded_size = tmp_dst_file.stat().st_size if remote_size != -1 and downloaded_size != remote_size: - raise Exception( - f"File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes" - ) + raise Exception(f'File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes') os.utime(tmp_dst_file, (remote_ts, remote_ts)) tmp_dst_file.chmod(0o644) tmp_dst_file.replace(dst_file) @@ -79,7 +71,8 @@ def downloading_worker(q): url, dst_file, working_dir, updated, remote_size = item - print("downloading", url, "to", dst_file.relative_to(working_dir), flush=True) + print("downloading", url, "to", + dst_file.relative_to(working_dir), flush=True) try: do_download(url, dst_file, updated, remote_size) except Exception: @@ -93,35 +86,30 @@ def downloading_worker(q): def create_workers(n): task_queue = queue.Queue() for i in range(n): - t = threading.Thread(target=downloading_worker, args=(task_queue,)) + t = threading.Thread(target=downloading_worker, args=(task_queue, )) t.start() return task_queue def ensure_safe_name(filename): - filename = filename.replace("\0", " ") - if filename == ".": - return " ." - elif filename == "..": - return ". ." + filename = filename.replace('\0', ' ') + if filename == '.': + return ' .' + elif filename == '..': + return '. .' else: - return filename.replace("/", "\\").replace("\\", "_") + return filename.replace('/', '\\').replace('\\', '_') def main(): import argparse - parser = argparse.ArgumentParser() parser.add_argument("--base-url", default=BASE_URL) parser.add_argument("--working-dir", default=WORKING_DIR) - parser.add_argument( - "--workers", default=1, type=int, help="number of concurrent downloading jobs" - ) - parser.add_argument( - "--fast-skip", - action="store_true", - help="do not verify size and timestamp of existing files", - ) + parser.add_argument("--workers", default=1, type=int, + help='number of concurrent downloading jobs') + parser.add_argument("--fast-skip", action='store_true', + help='do not verify size and timestamp of existing files') parser.add_argument("--config", default=CONFIG) args = parser.parse_args() @@ -136,15 +124,14 @@ def main(): with open(args.config, "r") as f: REPOS = json.load(f) - def download(release, release_dir, tarball=False): + def download(release, release_dir, tarball = False): global total_size if tarball: - url = release["tarball_url"] + url = release['tarball_url'] updated = datetime.strptime( - release["published_at"], "%Y-%m-%dT%H:%M:%SZ" - ).timestamp() - dst_file = release_dir / "repo-snapshot.tar.gz" + release['published_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp() + dst_file = release_dir / 'repo-snapshot.tar.gz' remote_filelist.append(dst_file.relative_to(working_dir)) if dst_file.is_file(): @@ -154,21 +141,19 @@ def download(release, release_dir, tarball=False): # tarball has no size information, use -1 to skip size check task_queue.put((url, dst_file, working_dir, updated, -1)) - for asset in release["assets"]: - url = asset["browser_download_url"] + for asset in release['assets']: + url = asset['browser_download_url'] updated = datetime.strptime( - asset["updated_at"], "%Y-%m-%dT%H:%M:%SZ" - ).timestamp() - dst_file = release_dir / ensure_safe_name(asset["name"]) + asset['updated_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp() + dst_file = release_dir / ensure_safe_name(asset['name']) remote_filelist.append(dst_file.relative_to(working_dir)) - remote_size = asset["size"] + remote_size = asset['size'] total_size += remote_size if dst_file.is_file(): if args.fast_skip: - print( - "fast skipping", dst_file.relative_to(working_dir), flush=True - ) + print("fast skipping", dst_file.relative_to( + working_dir), flush=True) continue else: stat = dst_file.stat() @@ -176,12 +161,10 @@ def download(release, release_dir, tarball=False): local_mtime = stat.st_mtime # print(f"{local_filesize} vs {asset['size']}") # print(f"{local_mtime} vs {updated}") - if ( - local_mtime > updated - or remote_size == local_filesize - and local_mtime == updated - ): - print("skipping", dst_file.relative_to(working_dir), flush=True) + if local_mtime > updated or \ + remote_size == local_filesize and local_mtime == updated: + print("skipping", dst_file.relative_to( + working_dir), flush=True) continue else: dst_file.parent.mkdir(parents=True, exist_ok=True) @@ -199,10 +182,10 @@ def link_latest(name, repo_dir): pass for cfg in REPOS: - flat = False # build a folder for each release - versions = 1 # keep only one release - tarball = False # do not download the tarball - prerelease = False # filter out pre-releases + flat = False # build a folder for each release + versions = 1 # keep only one release + tarball = False # do not download the tarball + prerelease = False # filter out pre-releases if isinstance(cfg, str): repo = cfg else: @@ -221,26 +204,25 @@ def link_latest(name, repo_dir): try: headers = {"Accept": "application/vnd.github+json"} - r = github_get( - f"{args.base_url}{repo}/releases?per_page=100", headers=headers - ) - r.raise_for_status() - releases = r.json() - - next_url = r.headers["link"] - if next_url: - for url_str in next_url.split(","): - r = github_get(url_str.split(";")[0].strip()[1:-1], headers=headers) - r.raise_for_status() - releases.extend(r.json()) + releases = [] + url_str = f"{args.base_url}{repo}/releases" + while url_str: + r = github_get(url_str, headers=headers) + r.raise_for_status() + releases.extend(r.json()) + next_url = [ s.split(";")[0].strip()[1:-1] for s in r.headers["link"].split(",") if s.split(";")[1].strip() == 'rel="next"' ] + if next_url: + url_str = next_url[0] + else: + url_str = None except: traceback.print_exc() break n_downloaded = 0 for release in releases: - if not release["draft"] and (prerelease or not release["prerelease"]): - name = ensure_safe_name(release["name"] or release["tag_name"]) + if not release['draft'] and (prerelease or not release['prerelease']): + name = ensure_safe_name(release['name'] or release['tag_name']) if len(name) == 0: print("Error: Unnamed release") continue @@ -265,7 +247,7 @@ def link_latest(name, repo_dir): if cleaning: local_filelist = [] - for local_file in working_dir.glob("**/*"): + for local_file in working_dir.glob('**/*'): if local_file.is_file(): local_filelist.append(local_file.relative_to(working_dir)) @@ -274,7 +256,7 @@ def link_latest(name, repo_dir): old_file = working_dir / old_file old_file.unlink() - for local_dir in working_dir.glob("*/*/*"): + for local_dir in working_dir.glob('*/*/*'): if local_dir.is_dir(): try: # remove empty dirs only @@ -284,9 +266,8 @@ def link_latest(name, repo_dir): print("Total size is", sizeof_fmt(total_size, suffix="")) - if __name__ == "__main__": main() -# vim: ts=4 sw=4 sts=4 expandtab +# vim: ts=4 sw=4 sts=4 expandtab \ No newline at end of file From e647cf614aefa2e20498fe675005fcc0d9ded5e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=AB=E6=8C=AF=E5=AE=87?= Date: Mon, 22 Jan 2024 16:52:46 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AF=B9release=E6=95=B0?= =?UTF-8?q?=E9=87=8F=E5=88=A4=E6=96=AD=EF=BC=8C=E9=81=BF=E5=85=8D=E4=B8=8D?= =?UTF-8?q?=E5=BF=85=E8=A6=81=E7=9A=84api=E8=AF=B7=E6=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- github-release.py | 120 +++++++++++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 49 deletions(-) diff --git a/github-release.py b/github-release.py index 1c74989..6e0d2ad 100755 --- a/github-release.py +++ b/github-release.py @@ -21,20 +21,21 @@ TIMEOUT_OPTION = (7, 10) total_size = 0 -def sizeof_fmt(num, suffix='iB'): - for unit in ['','K','M','G','T','P','E','Z']: + +def sizeof_fmt(num, suffix="iB"): + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: if abs(num) < 1024.0: return "%3.2f%s%s" % (num, unit, suffix) num /= 1024.0 - return "%.2f%s%s" % (num, 'Y', suffix) + return "%.2f%s%s" % (num, "Y", suffix) + # wrap around requests.get to use token if available def github_get(*args, **kwargs): - headers = kwargs['headers'] if 'headers' in kwargs else {} - if 'GITHUB_TOKEN' in os.environ: - headers['Authorization'] = 'token {}'.format( - os.environ['GITHUB_TOKEN']) - kwargs['headers'] = headers + headers = kwargs["headers"] if "headers" in kwargs else {} + if "GITHUB_TOKEN" in os.environ: + headers["Authorization"] = "token {}".format(os.environ["GITHUB_TOKEN"]) + kwargs["headers"] = headers return requests.get(*args, **kwargs) @@ -44,7 +45,12 @@ def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size: r.raise_for_status() tmp_dst_file = None try: - with tempfile.NamedTemporaryFile(prefix="." + dst_file.name + ".", suffix=".tmp", dir=dst_file.parent, delete=False) as f: + with tempfile.NamedTemporaryFile( + prefix="." + dst_file.name + ".", + suffix=".tmp", + dir=dst_file.parent, + delete=False, + ) as f: tmp_dst_file = Path(f.name) for chunk in r.iter_content(chunk_size=1024**2): if chunk: # filter out keep-alive new chunks @@ -53,7 +59,9 @@ def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size: # check for downloaded size downloaded_size = tmp_dst_file.stat().st_size if remote_size != -1 and downloaded_size != remote_size: - raise Exception(f'File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes') + raise Exception( + f"File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes" + ) os.utime(tmp_dst_file, (remote_ts, remote_ts)) tmp_dst_file.chmod(0o644) tmp_dst_file.replace(dst_file) @@ -71,8 +79,7 @@ def downloading_worker(q): url, dst_file, working_dir, updated, remote_size = item - print("downloading", url, "to", - dst_file.relative_to(working_dir), flush=True) + print("downloading", url, "to", dst_file.relative_to(working_dir), flush=True) try: do_download(url, dst_file, updated, remote_size) except Exception: @@ -86,30 +93,35 @@ def downloading_worker(q): def create_workers(n): task_queue = queue.Queue() for i in range(n): - t = threading.Thread(target=downloading_worker, args=(task_queue, )) + t = threading.Thread(target=downloading_worker, args=(task_queue,)) t.start() return task_queue def ensure_safe_name(filename): - filename = filename.replace('\0', ' ') - if filename == '.': - return ' .' - elif filename == '..': - return '. .' + filename = filename.replace("\0", " ") + if filename == ".": + return " ." + elif filename == "..": + return ". ." else: - return filename.replace('/', '\\').replace('\\', '_') + return filename.replace("/", "\\").replace("\\", "_") def main(): import argparse + parser = argparse.ArgumentParser() parser.add_argument("--base-url", default=BASE_URL) parser.add_argument("--working-dir", default=WORKING_DIR) - parser.add_argument("--workers", default=1, type=int, - help='number of concurrent downloading jobs') - parser.add_argument("--fast-skip", action='store_true', - help='do not verify size and timestamp of existing files') + parser.add_argument( + "--workers", default=1, type=int, help="number of concurrent downloading jobs" + ) + parser.add_argument( + "--fast-skip", + action="store_true", + help="do not verify size and timestamp of existing files", + ) parser.add_argument("--config", default=CONFIG) args = parser.parse_args() @@ -124,14 +136,15 @@ def main(): with open(args.config, "r") as f: REPOS = json.load(f) - def download(release, release_dir, tarball = False): + def download(release, release_dir, tarball=False): global total_size if tarball: - url = release['tarball_url'] + url = release["tarball_url"] updated = datetime.strptime( - release['published_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp() - dst_file = release_dir / 'repo-snapshot.tar.gz' + release["published_at"], "%Y-%m-%dT%H:%M:%SZ" + ).timestamp() + dst_file = release_dir / "repo-snapshot.tar.gz" remote_filelist.append(dst_file.relative_to(working_dir)) if dst_file.is_file(): @@ -141,19 +154,21 @@ def download(release, release_dir, tarball = False): # tarball has no size information, use -1 to skip size check task_queue.put((url, dst_file, working_dir, updated, -1)) - for asset in release['assets']: - url = asset['browser_download_url'] + for asset in release["assets"]: + url = asset["browser_download_url"] updated = datetime.strptime( - asset['updated_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp() - dst_file = release_dir / ensure_safe_name(asset['name']) + asset["updated_at"], "%Y-%m-%dT%H:%M:%SZ" + ).timestamp() + dst_file = release_dir / ensure_safe_name(asset["name"]) remote_filelist.append(dst_file.relative_to(working_dir)) - remote_size = asset['size'] + remote_size = asset["size"] total_size += remote_size if dst_file.is_file(): if args.fast_skip: - print("fast skipping", dst_file.relative_to( - working_dir), flush=True) + print( + "fast skipping", dst_file.relative_to(working_dir), flush=True + ) continue else: stat = dst_file.stat() @@ -161,10 +176,12 @@ def download(release, release_dir, tarball = False): local_mtime = stat.st_mtime # print(f"{local_filesize} vs {asset['size']}") # print(f"{local_mtime} vs {updated}") - if local_mtime > updated or \ - remote_size == local_filesize and local_mtime == updated: - print("skipping", dst_file.relative_to( - working_dir), flush=True) + if ( + local_mtime > updated + or remote_size == local_filesize + and local_mtime == updated + ): + print("skipping", dst_file.relative_to(working_dir), flush=True) continue else: dst_file.parent.mkdir(parents=True, exist_ok=True) @@ -182,10 +199,10 @@ def link_latest(name, repo_dir): pass for cfg in REPOS: - flat = False # build a folder for each release - versions = 1 # keep only one release - tarball = False # do not download the tarball - prerelease = False # filter out pre-releases + flat = False # build a folder for each release + versions = 1 # keep only one release + tarball = False # do not download the tarball + prerelease = False # filter out pre-releases if isinstance(cfg, str): repo = cfg else: @@ -210,10 +227,14 @@ def link_latest(name, repo_dir): r = github_get(url_str, headers=headers) r.raise_for_status() releases.extend(r.json()) - next_url = [ s.split(";")[0].strip()[1:-1] for s in r.headers["link"].split(",") if s.split(";")[1].strip() == 'rel="next"' ] + next_url = [ + s.split(";")[0].strip()[1:-1] + for s in r.headers["link"].split(",") + if s.split(";")[1].strip() == 'rel="next"' + ] if next_url: url_str = next_url[0] - else: + elif versions > 0 and len(releases) > versions: url_str = None except: traceback.print_exc() @@ -221,8 +242,8 @@ def link_latest(name, repo_dir): n_downloaded = 0 for release in releases: - if not release['draft'] and (prerelease or not release['prerelease']): - name = ensure_safe_name(release['name'] or release['tag_name']) + if not release["draft"] and (prerelease or not release["prerelease"]): + name = ensure_safe_name(release["name"] or release["tag_name"]) if len(name) == 0: print("Error: Unnamed release") continue @@ -247,7 +268,7 @@ def link_latest(name, repo_dir): if cleaning: local_filelist = [] - for local_file in working_dir.glob('**/*'): + for local_file in working_dir.glob("**/*"): if local_file.is_file(): local_filelist.append(local_file.relative_to(working_dir)) @@ -256,7 +277,7 @@ def link_latest(name, repo_dir): old_file = working_dir / old_file old_file.unlink() - for local_dir in working_dir.glob('*/*/*'): + for local_dir in working_dir.glob("*/*/*"): if local_dir.is_dir(): try: # remove empty dirs only @@ -266,8 +287,9 @@ def link_latest(name, repo_dir): print("Total size is", sizeof_fmt(total_size, suffix="")) + if __name__ == "__main__": main() -# vim: ts=4 sw=4 sts=4 expandtab \ No newline at end of file +# vim: ts=4 sw=4 sts=4 expandtab From 3f6cd38e2d42c2abddf2fc634c281c0701cf7b21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=AB=E6=8C=AF=E5=AE=87?= Date: Mon, 22 Jan 2024 17:52:38 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E8=BF=98=E5=8E=9F=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- github-release.py | 112 +++++++++++++++++++--------------------------- 1 file changed, 47 insertions(+), 65 deletions(-) diff --git a/github-release.py b/github-release.py index 6e0d2ad..7e67680 100755 --- a/github-release.py +++ b/github-release.py @@ -21,21 +21,20 @@ TIMEOUT_OPTION = (7, 10) total_size = 0 - -def sizeof_fmt(num, suffix="iB"): - for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: +def sizeof_fmt(num, suffix='iB'): + for unit in ['','K','M','G','T','P','E','Z']: if abs(num) < 1024.0: return "%3.2f%s%s" % (num, unit, suffix) num /= 1024.0 - return "%.2f%s%s" % (num, "Y", suffix) - + return "%.2f%s%s" % (num, 'Y', suffix) # wrap around requests.get to use token if available def github_get(*args, **kwargs): - headers = kwargs["headers"] if "headers" in kwargs else {} - if "GITHUB_TOKEN" in os.environ: - headers["Authorization"] = "token {}".format(os.environ["GITHUB_TOKEN"]) - kwargs["headers"] = headers + headers = kwargs['headers'] if 'headers' in kwargs else {} + if 'GITHUB_TOKEN' in os.environ: + headers['Authorization'] = 'token {}'.format( + os.environ['GITHUB_TOKEN']) + kwargs['headers'] = headers return requests.get(*args, **kwargs) @@ -45,12 +44,7 @@ def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size: r.raise_for_status() tmp_dst_file = None try: - with tempfile.NamedTemporaryFile( - prefix="." + dst_file.name + ".", - suffix=".tmp", - dir=dst_file.parent, - delete=False, - ) as f: + with tempfile.NamedTemporaryFile(prefix="." + dst_file.name + ".", suffix=".tmp", dir=dst_file.parent, delete=False) as f: tmp_dst_file = Path(f.name) for chunk in r.iter_content(chunk_size=1024**2): if chunk: # filter out keep-alive new chunks @@ -59,9 +53,7 @@ def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size: # check for downloaded size downloaded_size = tmp_dst_file.stat().st_size if remote_size != -1 and downloaded_size != remote_size: - raise Exception( - f"File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes" - ) + raise Exception(f'File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes') os.utime(tmp_dst_file, (remote_ts, remote_ts)) tmp_dst_file.chmod(0o644) tmp_dst_file.replace(dst_file) @@ -79,7 +71,8 @@ def downloading_worker(q): url, dst_file, working_dir, updated, remote_size = item - print("downloading", url, "to", dst_file.relative_to(working_dir), flush=True) + print("downloading", url, "to", + dst_file.relative_to(working_dir), flush=True) try: do_download(url, dst_file, updated, remote_size) except Exception: @@ -93,35 +86,30 @@ def downloading_worker(q): def create_workers(n): task_queue = queue.Queue() for i in range(n): - t = threading.Thread(target=downloading_worker, args=(task_queue,)) + t = threading.Thread(target=downloading_worker, args=(task_queue, )) t.start() return task_queue def ensure_safe_name(filename): - filename = filename.replace("\0", " ") - if filename == ".": - return " ." - elif filename == "..": - return ". ." + filename = filename.replace('\0', ' ') + if filename == '.': + return ' .' + elif filename == '..': + return '. .' else: - return filename.replace("/", "\\").replace("\\", "_") + return filename.replace('/', '\\').replace('\\', '_') def main(): import argparse - parser = argparse.ArgumentParser() parser.add_argument("--base-url", default=BASE_URL) parser.add_argument("--working-dir", default=WORKING_DIR) - parser.add_argument( - "--workers", default=1, type=int, help="number of concurrent downloading jobs" - ) - parser.add_argument( - "--fast-skip", - action="store_true", - help="do not verify size and timestamp of existing files", - ) + parser.add_argument("--workers", default=1, type=int, + help='number of concurrent downloading jobs') + parser.add_argument("--fast-skip", action='store_true', + help='do not verify size and timestamp of existing files') parser.add_argument("--config", default=CONFIG) args = parser.parse_args() @@ -136,15 +124,14 @@ def main(): with open(args.config, "r") as f: REPOS = json.load(f) - def download(release, release_dir, tarball=False): + def download(release, release_dir, tarball = False): global total_size if tarball: - url = release["tarball_url"] + url = release['tarball_url'] updated = datetime.strptime( - release["published_at"], "%Y-%m-%dT%H:%M:%SZ" - ).timestamp() - dst_file = release_dir / "repo-snapshot.tar.gz" + release['published_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp() + dst_file = release_dir / 'repo-snapshot.tar.gz' remote_filelist.append(dst_file.relative_to(working_dir)) if dst_file.is_file(): @@ -154,21 +141,19 @@ def download(release, release_dir, tarball=False): # tarball has no size information, use -1 to skip size check task_queue.put((url, dst_file, working_dir, updated, -1)) - for asset in release["assets"]: - url = asset["browser_download_url"] + for asset in release['assets']: + url = asset['browser_download_url'] updated = datetime.strptime( - asset["updated_at"], "%Y-%m-%dT%H:%M:%SZ" - ).timestamp() - dst_file = release_dir / ensure_safe_name(asset["name"]) + asset['updated_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp() + dst_file = release_dir / ensure_safe_name(asset['name']) remote_filelist.append(dst_file.relative_to(working_dir)) - remote_size = asset["size"] + remote_size = asset['size'] total_size += remote_size if dst_file.is_file(): if args.fast_skip: - print( - "fast skipping", dst_file.relative_to(working_dir), flush=True - ) + print("fast skipping", dst_file.relative_to( + working_dir), flush=True) continue else: stat = dst_file.stat() @@ -176,12 +161,10 @@ def download(release, release_dir, tarball=False): local_mtime = stat.st_mtime # print(f"{local_filesize} vs {asset['size']}") # print(f"{local_mtime} vs {updated}") - if ( - local_mtime > updated - or remote_size == local_filesize - and local_mtime == updated - ): - print("skipping", dst_file.relative_to(working_dir), flush=True) + if local_mtime > updated or \ + remote_size == local_filesize and local_mtime == updated: + print("skipping", dst_file.relative_to( + working_dir), flush=True) continue else: dst_file.parent.mkdir(parents=True, exist_ok=True) @@ -199,10 +182,10 @@ def link_latest(name, repo_dir): pass for cfg in REPOS: - flat = False # build a folder for each release - versions = 1 # keep only one release - tarball = False # do not download the tarball - prerelease = False # filter out pre-releases + flat = False # build a folder for each release + versions = 1 # keep only one release + tarball = False # do not download the tarball + prerelease = False # filter out pre-releases if isinstance(cfg, str): repo = cfg else: @@ -242,8 +225,8 @@ def link_latest(name, repo_dir): n_downloaded = 0 for release in releases: - if not release["draft"] and (prerelease or not release["prerelease"]): - name = ensure_safe_name(release["name"] or release["tag_name"]) + if not release['draft'] and (prerelease or not release['prerelease']): + name = ensure_safe_name(release['name'] or release['tag_name']) if len(name) == 0: print("Error: Unnamed release") continue @@ -268,7 +251,7 @@ def link_latest(name, repo_dir): if cleaning: local_filelist = [] - for local_file in working_dir.glob("**/*"): + for local_file in working_dir.glob('**/*'): if local_file.is_file(): local_filelist.append(local_file.relative_to(working_dir)) @@ -277,7 +260,7 @@ def link_latest(name, repo_dir): old_file = working_dir / old_file old_file.unlink() - for local_dir in working_dir.glob("*/*/*"): + for local_dir in working_dir.glob('*/*/*'): if local_dir.is_dir(): try: # remove empty dirs only @@ -287,9 +270,8 @@ def link_latest(name, repo_dir): print("Total size is", sizeof_fmt(total_size, suffix="")) - if __name__ == "__main__": main() -# vim: ts=4 sw=4 sts=4 expandtab +# vim: ts=4 sw=4 sts=4 expandtab \ No newline at end of file From efc22b53b963922d3d1de19c727255cb9ecceddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=AB=E6=8C=AF=E5=AE=87?= Date: Mon, 22 Jan 2024 18:07:40 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E4=BF=AE=E6=AD=A3release=E5=88=A4=E6=96=AD?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- github-release.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/github-release.py b/github-release.py index 7e67680..3471e58 100755 --- a/github-release.py +++ b/github-release.py @@ -215,9 +215,11 @@ def link_latest(name, repo_dir): for s in r.headers["link"].split(",") if s.split(";")[1].strip() == 'rel="next"' ] - if next_url: + if versions > 0 and len(releases) > versions: + url_str = None + elif next_url: url_str = next_url[0] - elif versions > 0 and len(releases) > versions: + else: url_str = None except: traceback.print_exc() From dfdc537c836ce2887d21a196bf91394b8ecb4f7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=AB=E6=8C=AF=E5=AE=87?= Date: Tue, 23 Jan 2024 19:31:10 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=AD=A3=E5=88=99?= =?UTF-8?q?=E5=8C=B9=E9=85=8D=E6=8F=90=E5=8F=96next=5Furl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- github-release.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/github-release.py b/github-release.py index 3471e58..9875f5a 100755 --- a/github-release.py +++ b/github-release.py @@ -8,7 +8,7 @@ from datetime import datetime import tempfile import json - +import re import requests @@ -206,15 +206,12 @@ def link_latest(name, repo_dir): headers = {"Accept": "application/vnd.github+json"} releases = [] url_str = f"{args.base_url}{repo}/releases" + pattern = re.compile(r'<(.*)>;\s*rel="next"') while url_str: r = github_get(url_str, headers=headers) r.raise_for_status() releases.extend(r.json()) - next_url = [ - s.split(";")[0].strip()[1:-1] - for s in r.headers["link"].split(",") - if s.split(";")[1].strip() == 'rel="next"' - ] + next_url = re.findall(pattern=pattern,string=r.headers["link"]) if versions > 0 and len(releases) > versions: url_str = None elif next_url: From 46c8aa3767e8d7e6c566dba248b061bf0c8ada3b Mon Sep 17 00:00:00 2001 From: jimorsm Date: Fri, 2 Feb 2024 03:56:24 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=AD=A3=E5=88=99?= =?UTF-8?q?=EF=BC=8C=E4=BD=BF=E7=94=A8=E6=9C=80=E7=9F=AD=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- github-release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-release.py b/github-release.py index 9875f5a..cebf23f 100755 --- a/github-release.py +++ b/github-release.py @@ -206,7 +206,7 @@ def link_latest(name, repo_dir): headers = {"Accept": "application/vnd.github+json"} releases = [] url_str = f"{args.base_url}{repo}/releases" - pattern = re.compile(r'<(.*)>;\s*rel="next"') + pattern = re.compile(r'.*<(.*?)>;\s*rel="next"') while url_str: r = github_get(url_str, headers=headers) r.raise_for_status()