diff --git a/.github/workflows/weread.yml b/.github/workflows/weread.yml index 595d032ec1863..c43333f0abeb2 100644 --- a/.github/workflows/weread.yml +++ b/.github/workflows/weread.yml @@ -3,7 +3,7 @@ name: weread note sync on: workflow_dispatch: schedule: - - cron: "0 0 * * *" + - cron: "0 * * * *" concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -33,6 +33,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + ref: dev/easonlee - name: Set up Python uses: actions/setup-python@v4 with: @@ -47,4 +49,4 @@ jobs: - name: weread sync run: | python -u scripts/weread.py - \ No newline at end of file + diff --git a/OUT_FOLDER/7d4e80c3-3325-43e5-a3d0-66461f739274.svg b/OUT_FOLDER/7d4e80c3-3325-43e5-a3d0-66461f739274.svg new file mode 100644 index 0000000000000..fd34078cc0fcb --- /dev/null +++ b/OUT_FOLDER/7d4e80c3-3325-43e5-a3d0-66461f739274.svg @@ -0,0 +1,2 @@ + +2024: 188 hoursJan2024-01-01 12.43 mins2024-01-02 5.85 mins2024-01-03 48.72 mins2024-01-04 78.82 mins2024-01-05 112.3 mins2024-01-06 131.28 mins2024-01-07 245.73 mins2024-01-08 86.78 mins2024-01-09 53.0 mins2024-01-10 46.38 mins2024-01-11 25.35 mins2024-01-12 36.22 mins2024-01-13 8.07 mins2024-01-14 31.98 mins2024-01-15 38.98 mins2024-01-16 46.9 mins2024-01-17 84.0 mins2024-01-18 63.45 mins2024-01-19 60.82 mins2024-01-20 78.48 mins2024-01-21 25.12 mins2024-01-22 11.02 mins2024-01-23 16.77 mins2024-01-24 4.8 mins2024-01-25 26.68 mins2024-01-26 14.18 mins2024-01-27 137.93 mins2024-01-282024-01-29 25.38 mins2024-01-30 14.43 mins2024-01-31 27.3 mins2024-02-01 16.05 mins2024-02-02 4.05 mins2024-02-032024-02-04 52.72 minsFeb2024-02-05 4.02 mins2024-02-06 15.22 mins2024-02-07 27.27 mins2024-02-08 90.52 mins2024-02-092024-02-102024-02-11 56.98 mins2024-02-12 164.35 mins2024-02-13 83.73 mins2024-02-142024-02-15 88.57 mins2024-02-16 88.28 mins2024-02-17 42.72 mins2024-02-18 9.6 mins2024-02-19 78.67 mins2024-02-20 1.28 mins2024-02-212024-02-22 0.07 mins2024-02-23 46.53 mins2024-02-242024-02-252024-02-26 23.38 mins2024-02-27 7.98 mins2024-02-28 47.83 mins2024-02-29 2.43 mins2024-03-01 78.35 mins2024-03-022024-03-03Mar2024-03-04 15.93 mins2024-03-05 10.4 mins2024-03-06 19.02 mins2024-03-07 0.03 mins2024-03-08 137.48 mins2024-03-09 107.87 mins2024-03-10 17.87 mins2024-03-11 8.12 mins2024-03-12 20.0 mins2024-03-13 16.37 mins2024-03-14 42.57 mins2024-03-15 6.63 mins2024-03-16 107.83 mins2024-03-17 43.87 mins2024-03-18 18.47 mins2024-03-19 7.32 mins2024-03-20 7.32 mins2024-03-21 2.37 mins2024-03-22 29.22 mins2024-03-23 8.62 mins2024-03-24 24.85 mins2024-03-25 0.05 mins2024-03-262024-03-272024-03-282024-03-292024-03-30 41.95 mins2024-03-31 6.78 minsApr2024-04-01 5.48 mins2024-04-022024-04-03 29.12 mins2024-04-04 34.83 mins2024-04-05 74.5 mins2024-04-062024-04-07 2.27 mins2024-04-08 23.78 mins2024-04-09 25.73 mins2024-04-10 43.42 mins2024-04-11 552.98 mins2024-04-12 143.52 mins2024-04-13 165.7 mins2024-04-14 395.9 mins2024-04-15 63.08 mins2024-04-16 12.9 mins2024-04-17 33.62 mins2024-04-18 31.87 mins2024-04-19 0.03 mins2024-04-202024-04-21 25.5 mins2024-04-22 72.22 mins2024-04-232024-04-24 21.73 mins2024-04-25 12.8 mins2024-04-26 96.6 mins2024-04-27 72.4 mins2024-04-282024-04-29 0.3 mins2024-04-30 9.67 mins2024-05-01 0.2 mins2024-05-022024-05-032024-05-04 28.05 mins2024-05-05 35.73 minsMay2024-05-06 35.2 mins2024-05-07 12.48 mins2024-05-08 63.42 mins2024-05-09 0.02 mins2024-05-10 1.22 mins2024-05-11 23.18 mins2024-05-12 42.98 mins2024-05-13 0.75 mins2024-05-14 11.8 mins2024-05-15 17.48 mins2024-05-16 17.37 mins2024-05-172024-05-18 1.4 mins2024-05-19 25.5 mins2024-05-20 23.83 mins2024-05-21 20.07 mins2024-05-22 3.45 mins2024-05-232024-05-242024-05-252024-05-26 5.15 mins2024-05-27 36.48 mins2024-05-28 0.22 mins2024-05-292024-05-30 2.53 mins2024-05-31 0.6 mins2024-06-01 82.42 mins2024-06-02 68.63 minsJun2024-06-03 93.28 mins2024-06-04 53.95 mins2024-06-05 3.17 mins2024-06-062024-06-07 74.78 mins2024-06-08 52.92 mins2024-06-09 11.45 mins2024-06-10 31.95 mins2024-06-11 0.02 mins2024-06-122024-06-132024-06-14 0.73 mins2024-06-15 54.38 mins2024-06-16 385.03 mins2024-06-17 147.55 mins2024-06-18 4.8 mins2024-06-19 271.65 mins2024-06-20 212.22 mins2024-06-21 11.45 mins2024-06-22 95.87 mins2024-06-23 140.22 mins2024-06-24 38.83 mins2024-06-25 45.1 mins2024-06-26 91.98 mins2024-06-27 117.35 mins2024-06-28 6.28 mins2024-06-29 142.62 mins2024-06-30 86.73 minsJul2024-07-01 168.48 mins2024-07-02 33.07 mins2024-07-03 35.87 mins2024-07-04 142.35 mins2024-07-05 3.87 mins2024-07-062024-07-07 11.0 mins2024-07-082024-07-09 40.08 mins2024-07-10 1.43 mins2024-07-11 55.1 mins2024-07-12 30.2 mins2024-07-13 129.87 mins2024-07-14 30.27 mins2024-07-15 42.62 mins2024-07-162024-07-17 1.27 mins2024-07-18 61.27 mins2024-07-19 216.08 mins2024-07-20 91.97 mins2024-07-212024-07-22 9.1 mins2024-07-23 67.3 mins2024-07-24 31.68 mins2024-07-252024-07-26 86.92 mins2024-07-27 84.47 mins2024-07-282024-07-292024-07-302024-07-312024-08-01 5.27 mins2024-08-02 5.88 mins2024-08-03 218.93 mins2024-08-04Aug2024-08-052024-08-062024-08-07 9.5 mins2024-08-08 90.65 mins2024-08-092024-08-10 19.8 mins2024-08-11 39.15 mins2024-08-12 62.78 mins2024-08-132024-08-14 0.73 mins2024-08-152024-08-16 27.68 mins2024-08-17 54.65 mins2024-08-18 22.65 mins2024-08-192024-08-20 17.67 mins2024-08-21 35.85 mins2024-08-22 46.05 mins2024-08-23 51.12 mins2024-08-24 115.88 mins2024-08-25 61.27 mins2024-08-26 0.53 mins2024-08-27 19.33 mins2024-08-28 2.43 mins2024-08-29 44.42 mins2024-08-30 103.15 mins2024-08-31 205.88 mins2024-09-01 58.17 minsSep2024-09-02 44.9 mins2024-09-03 19.28 mins2024-09-04 1.52 mins2024-09-05 13.47 mins2024-09-06 77.13 mins2024-09-07 115.75 mins2024-09-08 63.1 mins2024-09-09 23.95 mins2024-09-10 32.43 mins2024-09-11 14.78 mins2024-09-122024-09-13 9.07 mins2024-09-142024-09-152024-09-162024-09-17 143.88 mins2024-09-18 25.78 mins2024-09-192024-09-202024-09-212024-09-222024-09-232024-09-242024-09-252024-09-262024-09-272024-09-282024-09-292024-09-302024-10-012024-10-022024-10-032024-10-042024-10-052024-10-06Oct2024-10-072024-10-082024-10-092024-10-102024-10-112024-10-122024-10-132024-10-142024-10-152024-10-162024-10-172024-10-182024-10-192024-10-202024-10-212024-10-222024-10-232024-10-242024-10-252024-10-262024-10-272024-10-282024-10-292024-10-302024-10-312024-11-012024-11-022024-11-03Nov2024-11-042024-11-052024-11-062024-11-072024-11-082024-11-092024-11-102024-11-112024-11-122024-11-132024-11-142024-11-152024-11-162024-11-172024-11-182024-11-192024-11-202024-11-212024-11-222024-11-232024-11-242024-11-252024-11-262024-11-272024-11-282024-11-292024-11-302024-12-01Dec2024-12-022024-12-032024-12-042024-12-052024-12-062024-12-072024-12-082024-12-092024-12-102024-12-112024-12-122024-12-132024-12-142024-12-152024-12-162024-12-172024-12-182024-12-192024-12-202024-12-212024-12-222024-12-232024-12-242024-12-252024-12-262024-12-272024-12-282024-12-292024-12-302024-12-31 \ No newline at end of file diff --git a/README.md b/README.md index 13a7224b31ca4..5b03194498c0b 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ | 微信群 | QQ群 | | --- | --- | -|
|
| +|
|
| ## 捐赠 diff --git a/scripts/book.py b/scripts/book.py index 63ab90701fffa..46ad5e2a884f3 100644 --- a/scripts/book.py +++ b/scripts/book.py @@ -1,4 +1,5 @@ import argparse +import traceback from datetime import datetime, timedelta import os @@ -10,6 +11,8 @@ import utils from config import book_properties_type_dict, tz from retrying import retry +import json +from datetime import datetime TAG_ICON_URL = "https://www.notion.so/icons/tag_gray.svg" USER_ICON_URL = "https://www.notion.so/icons/user-circle-filled_gray.svg" @@ -48,6 +51,7 @@ def insert_book_to_notion(books, index, bookId): book["书架分类"] = archive_dict.get(bookId) if bookId in notion_books: book.update(notion_books.get(bookId)) + bookInfo = weread_api.get_bookinfo(bookId) if bookInfo != None: book.update(bookInfo) @@ -62,13 +66,13 @@ def insert_book_to_notion(books, index, bookId): markedStatus = book.get("markedStatus") status = "想读" if markedStatus == 4: - status = "已读" + status = "阅读完" elif book.get("readingTime", 0) >= 60: status = "在读" book["阅读状态"] = status book["阅读时长"] = book.get("readingTime") book["阅读天数"] = book.get("totalReadDay") - book["评分"] = book.get("newRating") + #book["评分"] = book.get("newRating") if book.get("newRatingDetail") and book.get("newRatingDetail").get("myRating"): book["我的评分"] = rating.get(book.get("newRatingDetail").get("myRating")) elif status == "已读": @@ -81,43 +85,53 @@ def insert_book_to_notion(books, index, bookId): elif book.get("readingBookDate"): date = book.get("readingBookDate") book["时间"] = date - book["开始阅读时间"] = book.get("beginReadingDate") - book["最后阅读时间"] = book.get("lastReadingDate") + book["类型"] = "书籍" + #book["开始阅读时间"] = book.get("beginReadingDate") + #book["最后阅读时间"] = book.get("lastReadingDate") cover = book.get("cover").replace("/s_", "/t7_") if not cover and not cover.strip() and not cover.startswith("http"): cover = BOOK_ICON_URL if bookId not in notion_books: isbn = book.get("isbn") - if isbn and isbn.strip(): - douban_url = get_douban_url(isbn) - if douban_url: - book["douban_url"] = douban_url - book["书名"] = book.get("title") + #链接timeout，先注释 + #if isbn and isbn.strip(): + # douban_url = get_douban_url(isbn) + # if douban_url: + # book["douban_url"] = douban_url + book["标题"] = book.get("title") book["BookId"] = book.get("bookId") book["ISBN"] = book.get("isbn") book["链接"] = utils.get_weread_url(bookId) - book["简介"] = book.get("intro") + #book["简介"] = book.get("intro") book["作者"] = [ notion_helper.get_relation_id( x, notion_helper.author_database_id, USER_ICON_URL ) for x in book.get("author").split(" ") ] - if book.get("categories"): - book["分类"] = [ - notion_helper.get_relation_id( - x.get("title"), notion_helper.category_database_id, TAG_ICON_URL - ) - for x in book.get("categories") - ] + #if book.get("categories"): + # book["分类"] = [ + # notion_helper.get_relation_id( + # x.get("title"), notion_helper.category_database_id, TAG_ICON_URL + # ) + # for x in book.get("categories") + # ] properties = utils.get_properties(book, book_properties_type_dict) + + #print(f"正在插入《{book.get('title')}》,一共{len(books)}本，当前是第{index+1}本。") + if not book.get("readDetail") or not book.get("readDetail").get("data"): + print(f"《{book.get('title')}》没有阅读记录，跳过") + return + if book.get("时间"): - notion_helper.get_date_relation( + #取书籍的阅读记录，可能有多天 + book['时间'] = [x['readDate'] for x in book.get('readDetail').get('data')] + notion_helper.get_date_relations( properties, - pendulum.from_timestamp(book.get("时间"), tz="Asia/Shanghai"), + #时间戳转换为北京时间 + [pendulum.from_timestamp(x, tz="Asia/Shanghai") for x in book.get("时间")] ) - print(f"正在插入《{book.get('title')}》,一共{len(books)}本，当前是第{index+1}本。") parent = {"database_id": notion_helper.book_database_id, "type": "database_id"} result = None if bookId in notion_books: @@ -142,6 +156,7 @@ def insert_book_to_notion(books, index, bookId): def insert_read_data(page_id, readTimes): readTimes = dict(sorted(readTimes.items())) filter = {"property": "书架", "relation": {"contains": page_id}} + #拿到当前书籍$page_id的所有阅读记录 results = notion_helper.query_all_by_book(notion_helper.read_database_id, filter) for result in results: timestamp = result.get("properties").get("时间戳").get("number") @@ -160,6 +175,7 @@ def insert_read_data(page_id, readTimes): insert_to_notion(None, int(key), value, page_id) +#插入或更新阅读记录 def insert_to_notion(page_id, timestamp, duration, book_database_id): parent = {"database_id": notion_helper.read_database_id, "type": "database_id"} properties = { @@ -186,18 +202,32 @@ def insert_to_notion(page_id, timestamp, duration, book_database_id): if __name__ == "__main__": + current_time = datetime.now() + print("开始同步阅读记录，当前时间: ", current_time) weread_api = WeReadApi() notion_helper = NotionHelper() notion_books = notion_helper.get_all_book() bookshelf_books = weread_api.get_bookshelf() + # 有阅读记录的图书信息 + # 样例数据：{'$bookId': {'bookId': '26062915', 'progress': 1, 'chapterUid': 8, 'chapterOffset': 0, + # 'chapterIdx': 8, 'appId'： '11413501'， 'updateTime': 1691221509, 'readingTime': 875, + # 'syncKey': 720606794} + # } bookProgress = bookshelf_books.get("bookProgress") bookProgress = {book.get("bookId"): book for book in bookProgress} archive_dict = {} + #acchive是书单的名字 for archive in bookshelf_books.get("archive"): name = archive.get("name") bookIds = archive.get("bookIds") archive_dict.update({bookId: name for bookId in bookIds}) not_need_sync = [] + # Notion里的文献笔记 + # 样例数据：{'$bookId': {'pageId': '10c911dc-da56-8106-8b58-ddaacc287c74', 'readingTime': 2325, + # 'category': None, 'Sort': None, 'douban_url': None, + # 'cover': {'type': 'external', 'external': {'url': 'https://bts-image.xyzcdn.net/aHR0cHM6Ly9pbWFnZS54eXpjZG4ubmV0L0ZxUWs2VThtWDU0YnZ3MFBsbm5HemtHMVpEajkuanBn.jpg'}}, + # 'myRating': None, 'comment': None, 'status': '阅读完'} + # } for key, value in notion_books.items(): if ( ( @@ -205,17 +235,45 @@ def insert_to_notion(page_id, timestamp, duration, book_database_id): or value.get("readingTime") == bookProgress.get(key).get("readingTime") ) and (archive_dict.get(key) == value.get("category")) - and (value.get("cover") is not None) - and ( - value.get("status") != "已读" - or (value.get("status") == "已读" and value.get("myRating")) - ) + #and (value.get("cover") is not None) + #and ( + # value.get("status") != "已读" + # or (value.get("status") == "已读" and value.get("myRating")) + #) ): + #这里判定Notion中的书籍和微信中的书籍是否有属性发现变化，没有的话就不需要同步了 not_need_sync.append(key) + #continue notebooks = weread_api.get_notebooklist() + notebooks_map = {d["bookId"] : d for d in notebooks if "bookId" in d} notebooks = [d["bookId"] for d in notebooks if "bookId" in d] + books = bookshelf_books.get("books") + bookshelf_map = {d["bookId"] : d for d in books if "bookId" in d} books = [d["bookId"] for d in books if "bookId" in d] + + books = list((set(notebooks) | set(books)) - set(not_need_sync)) + len = len(books) for index, bookId in enumerate(books): - insert_book_to_notion(books, index, bookId) + try: + bp = bookProgress[bookId] if bookId in bookProgress else None + bs = bookshelf_map[bookId] if bookId in bookshelf_map else None + nt = notebooks_map[bookId] if bookId in notebooks_map else None + + title = bs['title'] if bs and 'title' in bs else "" + print(f"正在插入《{title}》,一共{len}本，当前是第{index + 1}本。") + + if bp and bp['readingTime'] < 600: + print(f"《{title}》阅读不超时10分钟，跳过") + continue + + if not nt: + print(f"《{title}》没有阅读笔记，跳过") + continue + + insert_book_to_notion(books, index, bookId) + except Exception as e: + print("处理book: " + bookId + "出现异常，跳过:") + traceback.print_exc() + continue \ No newline at end of file diff --git a/scripts/config.py b/scripts/config.py index df0985b1b98d0..cc57f7889ebcb 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -10,7 +10,7 @@ SELECT = "select" book_properties_type_dict = { - "书名":TITLE, + "标题":TITLE, "BookId":RICH_TEXT, "ISBN":RICH_TEXT, "链接":URL, @@ -30,5 +30,6 @@ "书架分类":SELECT, "我的评分":SELECT, "豆瓣链接":URL, + "类型":SELECT } tz='Asia/Shanghai' \ No newline at end of file diff --git a/scripts/notion_helper.py b/scripts/notion_helper.py index c6fc0de1f67d7..7e554875084e4 100644 --- a/scripts/notion_helper.py +++ b/scripts/notion_helper.py @@ -5,7 +5,7 @@ from notion_client import Client from retrying import retry -from datetime import timedelta +from datetime import timedelta, datetime from dotenv import load_dotenv from utils import ( format_date, @@ -28,10 +28,13 @@ TARGET_ICON_URL = "https://www.notion.so/icons/target_red.svg" BOOKMARK_ICON_URL = "https://www.notion.so/icons/bookmark_gray.svg" +DATE_EMOJ_ICON = "🗓️" + + class NotionHelper: database_name_dict = { - "BOOK_DATABASE_NAME": "书架", + "BOOK_DATABASE_NAME": "文献笔记", "REVIEW_DATABASE_NAME": "笔记", "BOOKMARK_DATABASE_NAME": "划线", "DAY_DATABASE_NAME": "日", @@ -47,6 +50,7 @@ class NotionHelper: heatmap_block_id = None def __init__(self): + print("notion helper开始初始化", datetime.now()) self.client = Client(auth=os.getenv("NOTION_TOKEN"), log_level=logging.ERROR) self.__cache = {} self.page_id = self.extract_page_id(os.getenv("NOTION_PAGE")) @@ -90,6 +94,7 @@ def __init__(self): self.update_book_database() if self.read_database_id is None: self.create_database() + print("notion helper完成初始化", datetime.now()) def extract_page_id(self, notion_url): # 正则表达式匹配 32 个字符的 Notion page_id @@ -108,7 +113,7 @@ def search_database(self, block_id): for child in children: # 检查子块的类型 if child["type"] == "child_database": - self.database_id_dict[child.get("child_database").get("title")] = ( + self.database_id_dict[child.get("child_database").get("title").strip()] = ( child.get("id") ) elif child["type"] == "embed" and child.get("embed").get("url"): @@ -215,6 +220,14 @@ def get_year_relation_id(self, date): ) def get_day_relation_id(self, date): + new_date = date.replace(hour=0, minute=0, second=0, microsecond=0) + day = new_date.strftime("%Y-%m-%d") + + properties = {} + return self.get_reltion_id_by_property( + "【兼容】日期", day, "date", self.day_database_id, DATE_EMOJ_ICON, properties + ) + def get_day_relation_id_old(self, date): new_date = date.replace(hour=0, minute=0, second=0, microsecond=0) timestamp = (new_date - timedelta(hours=8)).timestamp() day = new_date.strftime("%Y年%m月%d日") @@ -258,6 +271,25 @@ def get_relation_id(self, name, id, icon, properties={}): self.__cache[key] = page_id return page_id + def get_reltion_id_by_property(self, property_name, property_value, property_type, + id, icon, properties = {}): + key = f"{id}{property_name}-{property_value}" + if key in self.__cache: + return self.__cache.get(key) + filter = {"property": property_name, property_type: {"equals": property_value}} + response = self.client.databases.query(database_id=id, filter=filter) + if len(response.get("results")) == 0: + raise Exception("未找到该日期") + #parent = {"database_id": id, "type": "database_id"} + #properties["标题"] = get_title(name) + #page_id = self.client.pages.create( + # parent=parent, properties=properties, icon=get_icon(icon) + #).get("id") + else: + page_id = response.get("results")[0].get("id") + self.__cache[key] = page_id + return page_id + def insert_bookmark(self, id, bookmark): icon = get_icon(BOOKMARK_ICON_URL) properties = { @@ -369,7 +401,9 @@ def delete_block(self, block_id): @retry(stop_max_attempt_number=3, wait_fixed=5000) def get_all_book(self): """从Notion中获取所有的书籍""" + print("从notion拉取所有书籍开始： ", datetime.now()) results = self.query_all(self.book_database_id) + #filter = {"property": "类型", "relation": {"contains": page_id}} books_dict = {} for result in results: bookId = get_property_value(result.get("properties").get("BookId")) @@ -385,13 +419,14 @@ def get_all_book(self): "douban_url": get_property_value( result.get("properties").get("豆瓣链接") ), - "cover": result.get("cover"), + "cover": result.get("cover"),#Notion里已无这个字段, "myRating": get_property_value( result.get("properties").get("我的评分") ), "comment": get_property_value(result.get("properties").get("豆瓣短评")), "status": get_property_value(result.get("properties").get("阅读状态")), } + print("从notion拉取所有书籍结束： ", datetime.now()) return books_dict @retry(stop_max_attempt_number=3, wait_fixed=5000) @@ -418,6 +453,7 @@ def query_all(self, database_id): has_more = True start_cursor = None while has_more: + print("从notion拉取数据...", datetime.now()) response = self.client.databases.query( database_id=database_id, start_cursor=start_cursor, @@ -428,23 +464,30 @@ def query_all(self, database_id): results.extend(response.get("results")) return results - def get_date_relation(self, properties, date): - properties["年"] = get_relation( - [ - self.get_year_relation_id(date), - ] - ) - properties["月"] = get_relation( + def get_date_relations(self, properties, dates): + properties["阅读日"] = get_relation( [ - self.get_month_relation_id(date), + self.get_day_relation_id(date) for date in dates ] ) - properties["周"] = get_relation( - [ - self.get_week_relation_id(date), - ] - ) - properties["日"] = get_relation( + + def get_date_relation(self, properties, date): + #properties["年"] = get_relation( + # [ + # self.get_year_relation_id(date), + # ] + #) + #properties["月"] = get_relation( + # [ + # self.get_month_relation_id(date), + # ] + #) + #properties["周"] = get_relation( + # [ + # self.get_week_relation_id(date), + # ] + # + properties["阅读日"] = get_relation( [ self.get_day_relation_id(date), ] diff --git a/scripts/read_time.py b/scripts/read_time.py index ad84e1e949af4..8a3689b0d1fc6 100644 --- a/scripts/read_time.py +++ b/scripts/read_time.py @@ -82,20 +82,23 @@ def get_file(): HEATMAP_GUIDE = "https://mp.weixin.qq.com/s?__biz=MzI1OTcxOTI4NA==&mid=2247484145&idx=1&sn=81752852420b9153fc292b7873217651&chksm=ea75ebeadd0262fc65df100370d3f983ba2e52e2fcde2deb1ed49343fbb10645a77570656728&token=157143379&lang=zh_CN#rd" if __name__ == "__main__": + os.environ['NOTION_PAGE'] = 'Life-OS-System-81b25bbd21684a3c943eb03a44b2f900' + os.environ['NOTION_TOKEN'] = 'secret_LAVFOz2S74ztgXTaDPuYKEjGEKPtflnk5IV0DbzW4vd' + os.environ['WEREAD_COOKIE'] = 'ptcz=754d621bb3cc45abb68f07d842270b8db5224d5c18bc38c78d239f19edb36567; pgv_pvid=6811236617; _qimei_uuid42=17c02151133100c79dcf768330e7e363052c0fa717; _qimei_q36=; wr_gid=273651192; wr_theme=white; RK=QHthLXYzU2; pac_uid=0_3fbe47a232d9f; iip=0; suid=ek171315182328217824; wr_vid=68910221; wr_pf=0; wr_rt=web%40xEmeptoh5f57cXUSQSQ_AL; wr_localvid=be8325e0741b7c8dbe86f3b; wr_name=Easonlee; wr_gender=1; wr_avatar=https%3A%2F%2Fwx.qlogo.cn%2Fmmhead%2FcyiaTKm65RvUN6Mnm27TwKefnx3fCSsEClhwOuysFbHw%2F0; qq_domain_video_guid_verify=aef172aa1a598b90; _ga=GA1.2.1031625882.1723940499; _ga_8YVFNWD1KC=GS1.2.1723940499.1.1.1723940531.0.0.0; _clck=3911196076|1|fon|0; _qimei_fingerprint=88a18db09377adee6270beae94358cff; _qimei_h38=b115791b9dcf768330e7e36303000006617c02; wr_fp=25923672; wr_skey=7SD1H2_a' notion_helper = NotionHelper() weread_api = WeReadApi() - image_file = get_file() - if image_file: - image_url = f"https://raw.githubusercontent.com/{os.getenv('REPOSITORY')}/{os.getenv('REF').split('/')[-1]}/OUT_FOLDER/{image_file}" - heatmap_url = f"https://heatmap.malinkang.com/?image={image_url}" - if notion_helper.heatmap_block_id: - response = notion_helper.update_heatmap( - block_id=notion_helper.heatmap_block_id, url=heatmap_url - ) - else: - print(f"更新热力图失败，没有添加热力图占位。具体参考：{HEATMAP_GUIDE}") - else: - print(f"更新热力图失败，没有生成热力图。具体参考：{HEATMAP_GUIDE}") + #image_file = get_file() + #if image_file: + # image_url = f"https://raw.githubusercontent.com/{os.getenv('REPOSITORY')}/{os.getenv('REF').split('/')[-1]}/OUT_FOLDER/{image_file}" + # heatmap_url = f"https://heatmap.malinkang.com/?image={image_url}" + # if notion_helper.heatmap_block_id: + # response = notion_helper.update_heatmap( + # block_id=notion_helper.heatmap_block_id, url=heatmap_url + # ) + # else: + # print(f"更新热力图失败，没有添加热力图占位。具体参考：{HEATMAP_GUIDE}") + #else: + # print(f"更新热力图失败，没有生成热力图。具体参考：{HEATMAP_GUIDE}") api_data = weread_api.get_api_data() readTimes = {int(key): value for key, value in api_data.get("readTimes").items()} now = pendulum.now("Asia/Shanghai").start_of("day") diff --git a/scripts/utils.py b/scripts/utils.py index 7682a2b749371..60ad98e92ebfc 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -88,7 +88,10 @@ def get_date(start, end=None): def get_icon(url): - return {"type": "external", "external": {"url": url}} + if url.find("https") != -1: + return {"type": "external", "external": {"url": url}} + else: + return {"type": "emoji", 'emoji': url} def get_select(name): diff --git a/scripts/weread.py b/scripts/weread.py index ee70132671881..c021e7323ba0c 100644 --- a/scripts/weread.py +++ b/scripts/weread.py @@ -4,6 +4,7 @@ from notion_helper import NotionHelper from weread_api import WeReadApi +from datetime import datetime from utils import ( get_callout, @@ -16,6 +17,9 @@ ) +# 返回微信阅读最新的划线信息，会做如下两个处理： +# a）如果划线在notion中存在，则设置blockid +# b）如果划线在notion存在，但微信阅读已经不存在了，则从notion中删除。 def get_bookmark_list(page_id, bookId): """获取我的划线""" filter = { @@ -38,6 +42,7 @@ def get_bookmark_list(page_id, bookId): for i in bookmarks: if i.get("bookmarkId") in dict1: i["blockId"] = dict1.pop(i.get("bookmarkId")) + # 删除微信阅读上已经取消的划线 for blockId in dict1.values(): notion_helper.delete_block(blockId) notion_helper.delete_block(dict2.get(blockId)) @@ -64,6 +69,7 @@ def get_review_list(page_id,bookId): for i in reviews: if i.get("reviewId") in dict1: i["blockId"] = dict1.pop(i.get("reviewId")) + # 删除微信阅读已经取消的笔记 for blockId in dict1.values(): notion_helper.delete_block(blockId) notion_helper.delete_block(dict2.get(blockId)) @@ -127,7 +133,7 @@ def download_image(url, save_dir="cover"): def sort_notes(page_id, chapter, bookmark_list): - """对笔记进行排序""" + """对笔记和划线进行排序""" bookmark_list = sorted( bookmark_list, key=lambda x: ( @@ -141,45 +147,52 @@ def sort_notes(page_id, chapter, bookmark_list): notes = [] if chapter != None: filter = {"property": "书籍", "relation": {"contains": page_id}} + + # 已经插入到notion的chapter信息: results = notion_helper.query_all_by_book( notion_helper.chapter_database_id, filter ) - dict1 = { + chapterUId2BlockIdMap = { get_number_from_result(x, "chapterUid"): get_rich_text_from_result( x, "blockId" ) for x in results } - dict2 = {get_rich_text_from_result(x, "blockId"): x.get("id") for x in results} - d = {} + blockId2ChapterMap = {get_rich_text_from_result(x, "blockId"): x.get("id") for x in results} + # 按chapter聚合的划线和笔记 + notesByChapter = {} for data in bookmark_list: chapterUid = data.get("chapterUid", 1) - if chapterUid not in d: - d[chapterUid] = [] - d[chapterUid].append(data) - for key, value in d.items(): + if chapterUid not in notesByChapter: + notesByChapter[chapterUid] = [] + notesByChapter[chapterUid].append(data) + + # 把章节信息也插入到notes中 + for key, value in notesByChapter.items(): if key in chapter: - if key in dict1: - chapter.get(key)["blockId"] = dict1.pop(key) + if key in chapterUId2BlockIdMap: + chapter.get(key)["blockId"] = chapterUId2BlockIdMap.pop(key) notes.append(chapter.get(key)) notes.extend(value) - for blockId in dict1.values(): + + for blockId in chapterUId2BlockIdMap.values(): notion_helper.delete_block(blockId) - notion_helper.delete_block(dict2.get(blockId)) + notion_helper.delete_block(blockId2ChapterMap.get(blockId)) else: notes.extend(bookmark_list) return notes -def append_blocks(id, contents): +def append_blocks(pageId, contents): print(f"笔记数{len(contents)}") before_block_id = "" - block_children = notion_helper.get_block_children(id) + block_children = notion_helper.get_block_children(pageId) + # 先确保第一个block是toc，如果不是就插入一个 if len(block_children) > 0 and block_children[0].get("type") == "table_of_contents": before_block_id = block_children[0].get("id") else: response = notion_helper.append_blocks( - block_id=id, children=[get_table_of_contents()] + block_id=pageId, children=[get_table_of_contents()] ) before_block_id = response.get("results")[0].get("id") blocks = [] @@ -187,7 +200,7 @@ def append_blocks(id, contents): l = [] for content in contents: if len(blocks) == 100: - results = append_blocks_to_notion(id, blocks, before_block_id, sub_contents) + results = append_blocks_to_notion(pageId, blocks, before_block_id, sub_contents) before_block_id = results[-1].get("blockId") l.extend(results) blocks.clear() @@ -197,7 +210,7 @@ def append_blocks(id, contents): elif "blockId" in content: if len(blocks) > 0: l.extend( - append_blocks_to_notion(id, blocks, before_block_id, sub_contents) + append_blocks_to_notion(pageId, blocks, before_block_id, sub_contents) ) blocks.clear() sub_contents.clear() @@ -207,15 +220,15 @@ def append_blocks(id, contents): sub_contents.append(content) if len(blocks) > 0: - l.extend(append_blocks_to_notion(id, blocks, before_block_id, sub_contents)) + l.extend(append_blocks_to_notion(pageId, blocks, before_block_id, sub_contents)) for index, value in enumerate(l): print(f"正在插入第{index+1}条笔记，共{len(l)}条") if "bookmarkId" in value: - notion_helper.insert_bookmark(id, value) + notion_helper.insert_bookmark(pageId, value) elif "reviewId" in value: - notion_helper.insert_review(id, value) + notion_helper.insert_review(pageId, value) else: - notion_helper.insert_chapter(id, value) + notion_helper.insert_chapter(pageId, value) def content_to_block(content): @@ -255,6 +268,8 @@ def append_blocks_to_notion(id, blocks, after, contents): if __name__ == "__main__": + current_time = datetime.now() + print("开始同步笔记，当前时间: ", current_time) parser = argparse.ArgumentParser() options = parser.parse_args() branch = os.getenv("REF").split("/")[-1] @@ -263,7 +278,8 @@ def append_blocks_to_notion(id, blocks, after, contents): notion_helper = NotionHelper() notion_books = notion_helper.get_all_book() books = weread_api.get_notebooklist() - print(len(books)) + #print(notion_books) + #print(books) if books != None: for index, book in enumerate(books): bookId = book.get("bookId") @@ -271,6 +287,7 @@ def append_blocks_to_notion(id, blocks, after, contents): sort = book.get("sort") if bookId not in notion_books: continue + # 这个sort是啥意思？ if sort == notion_books.get(bookId).get("Sort"): continue pageId = notion_books.get(bookId).get("pageId") @@ -279,6 +296,7 @@ def append_blocks_to_notion(id, blocks, after, contents): bookmark_list = get_bookmark_list(pageId, bookId) reviews = get_review_list(pageId,bookId) bookmark_list.extend(reviews) + #会把原来页面的所有自动同步的block删除（包括笔记、划线、章节） content = sort_notes(pageId, chapter, bookmark_list) append_blocks(pageId, content) properties = {