add multi thread

Signed-off-by: ふぁ <yuki@yuki0311.com>

test/python/test_serialize.py

@@ -7,6 +7,8 @@ from pathlib import Path
 import time
 import glob
 import aenum
+import concurrent.futures
+
 
 logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s")
 logger = logging.getLogger("test_serialize")
@@ -17,22 +19,6 @@ SLEEP_TIME = float(os.environ.get("SLEEP", "0"))
 CUESOR_TEST_COUNT = int(os.environ.get("CUESOR_TEST_COUNT", "3"))
 STRICT_MODE = os.environ.get("STRICT_MODE", "false").lower() == "true"
 
-if Path("cookie.json").exists():
-    with open("cookie.json", "r") as f:
-        cookies = json.load(f)
-elif TWITTER_SESSION is not None:
-    data = base64.b64decode(TWITTER_SESSION).decode("utf-8")
-    cookies = json.loads(data)
-else:
-    commands = ["python -m pip install tweepy_authlib", "python tools/login.py"]
-    raise Exception(f'cookie.json not found. Please run `{"; ".join(commands)}` first.')
-
-cookies_str = "; ".join([f"{k}={v}" for k, v in cookies.items()])
-
-
-with open("src/config/placeholder.json", "r") as f:
-    placeholder = json.load(f)
-
 
 def get_key(snake_str):
     components = snake_str.split("_")
@@ -120,15 +106,15 @@ def save_cache(data):
         json.dump(data, f, indent=4)
 
 
-# ====== Load cache ======
-for file in glob.glob("cache/*.json"):
+def task_callback(file):
+    try:
         with open(file, "r") as f:
             cache = json.load(f)
         data = pt.__dict__[cache["type"]].from_json(cache["raw"])
         rate = match_rate(data.to_dict(), json.loads(cache["raw"]))
-    logger.info(f"Match rate: {rate}")
-
-logger.info(f"Success: {len(glob.glob('cache/*.json'))}")
+        return f"Match rate: {rate}"
+    except Exception as e:
+        return f"Error: {e} {file}"
 
 
 def error_dump(e):
@@ -142,6 +128,31 @@ def error_dump(e):
     logger.info("================================")
 
 
+if __name__ == "__main__":
+    if Path("cookie.json").exists():
+        with open("cookie.json", "r") as f:
+            cookies = json.load(f)
+    elif TWITTER_SESSION is not None:
+        data = base64.b64decode(TWITTER_SESSION).decode("utf-8")
+        cookies = json.loads(data)
+    else:
+        commands = ["python -m pip install tweepy_authlib", "python tools/login.py"]
+        raise Exception(
+            f'cookie.json not found. Please run `{"; ".join(commands)}` first.'
+        )
+
+    cookies_str = "; ".join([f"{k}={v}" for k, v in cookies.items()])
+
+    with open("src/config/placeholder.json", "r") as f:
+        placeholder = json.load(f)
+
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        tasks = [executor.submit(task_callback, x) for x in glob.glob("cache/*.json")]
+        for task in concurrent.futures.as_completed(tasks):
+            logger.info(task.result())
+
+    logger.info(f"Success: {len(glob.glob('cache/*.json'))}")
+
     api_conf = pt.Configuration(
         api_key={
             "ClientLanguage": "en",
@@ -175,11 +186,18 @@ for x in [pt.DefaultApi, pt.TweetApi, pt.UserApi, pt.UsersApi, pt.UserListApi]:
                     cursor_history.add(cursor)
                     logger.info(f"Try: {key} {cursor}")
 
-                kwargs = get_kwargs(key, {} if cursor is None else {"cursor": cursor})
+                    kwargs = get_kwargs(
+                        key, {} if cursor is None else {"cursor": cursor}
+                    )
                     res: pt.ApiResponse = getattr(x(api_client), props)(**kwargs)
                     data = res.data.to_dict()
 
-                save_cache({"raw": res.raw_data, "type": res.data.__class__.__name__})
+                    save_cache(
+                        {
+                            "raw": res.raw_data,
+                            "type": res.data.__class__.__name__,
+                        }
+                    )
 
                     new_cursor = set(get_cursor(data, find_cursor)) - cursor_history
                     cursor_list.update(new_cursor)
@@ -199,7 +217,6 @@ for x in [pt.DefaultApi, pt.TweetApi, pt.UserApi, pt.UsersApi, pt.UserListApi]:
                 error_dump(e)
                 error_count += 1
 
-
     try:
         logger.info(f"Try: Self UserByScreenName Test")
         kwargs = get_kwargs("UserByScreenName", {"screen_name": "a810810931931"})
@@ -208,7 +225,8 @@ try:
 
         rate = match_rate(data, json.loads(res.raw_data))
         logger.info(f"Match rate: {rate}")
-    if not data["data"]["user"]["result"]["legacy"]["screen_name"] == "a810810931931":
+        screen_name = data["data"]["user"]["result"]["legacy"]["screen_name"]
+        if not screen_name == "a810810931931":
             raise Exception("UserByScreenName failed")
     except Exception as e:
         error_dump(e)