Merge branch 'dev' into remove-agpl

Signed-off-by: ふぁ <yuki@yuki0311.com>

dist/compatible/schemas/content.yaml

@@ -1,5 +1,14 @@
 components:
   schemas:
+    ClientEventInfo:
+      properties:
+        component:
+          type: string
+        details:
+          additionalProperties: true
+          type: object
+        element:
+          type: string
     ContentEntryType:
       enum:
       - TimelineTimelineItem
@@ -54,13 +63,11 @@ components:
           $ref: ./typename.yaml#/components/schemas/TypeName
         result:
           $ref: ./tweet.yaml#/components/schemas/TweetUnion
-      required:
+      required: null
-      - result
     ModuleEntry:
       properties:
         clientEventInfo:
-          additionalProperties: true
+          $ref: '#/components/schemas/ClientEventInfo'
-          type: object
         itemContent:
           $ref: '#/components/schemas/ItemContentUnion'
       required:
@@ -116,8 +123,7 @@ components:
         __typename:
           $ref: ./typename.yaml#/components/schemas/TypeName
         clientEventInfo:
-          additionalProperties: true
+          $ref: '#/components/schemas/ClientEventInfo'
-          type: object
         entryType:
           $ref: '#/components/schemas/ContentEntryType'
         feedbackInfo:

dist/compatible/schemas/instruction.yaml

@@ -1,5 +1,42 @@
 components:
   schemas:
+    Callback:
+      properties:
+        endpoint:
+          format: uri
+          type: string
+      required:
+      - endpoint
+    CoverCta:
+      properties:
+        Text:
+          type: string
+        buttonStyle:
+          enum:
+          - Primary
+          type: string
+        callbacks:
+          items:
+            $ref: '#/components/schemas/Callback'
+          type: array
+        clientEventInfo:
+          $ref: '#/components/schemas/CtaClientEventInfo'
+        ctaBehavior:
+          $ref: '#/components/schemas/TimelineCoverBehavior'
+      required:
+      - text
+      - ctaBehavior
+      - callbacks
+      - clientEventInfo
+      - buttonStyle
+    CtaClientEventInfo:
+      properties:
+        action:
+          enum:
+          - primary_cta
+          type: string
+      required:
+      - action
     InstructionType:
       enum:
       - TimelineAddEntries
@@ -9,6 +46,7 @@ components:
       - TimelineReplaceEntry
       - TimelineShowAlert
       - TimelineTerminateTimeline
+      - TimelineShowCover
       type: string
     InstructionUnion:
       discriminator:
@@ -19,6 +57,7 @@ components:
           TimelinePinEntry: '#/components/schemas/TimelinePinEntry'
           TimelineReplaceEntry: '#/components/schemas/TimelineReplaceEntry'
           TimelineShowAlert: '#/components/schemas/TimelineShowAlert'
+          TimelineShowCover: '#/components/schemas/TimelineShowCover'
           TimelineTerminateTimeline: '#/components/schemas/TimelineTerminateTimeline'
         propertyName: type
       oneOf:
@@ -29,6 +68,47 @@ components:
       - $ref: '#/components/schemas/TimelineReplaceEntry'
       - $ref: '#/components/schemas/TimelineShowAlert'
       - $ref: '#/components/schemas/TimelineTerminateTimeline'
+      - $ref: '#/components/schemas/TimelineShowCover'
+    Text:
+      properties:
+        entities:
+          items:
+            $ref: '#/components/schemas/TextEntity'
+          type: array
+        text:
+          type: string
+      required:
+      - text
+      - entities
+    TextEntity:
+      properties:
+        fromIndex:
+          type: integer
+        ref:
+          $ref: '#/components/schemas/TextEntityRef'
+        toIndex:
+          type: integer
+      required:
+      - fromIndex
+      - toIndex
+      - ref
+    TextEntityRef:
+      properties:
+        type:
+          enum:
+          - TimelineUrl
+          type: string
+        url:
+          format: uri
+          type: string
+        urlType:
+          enum:
+          - ExternalUrl
+          type: string
+      required:
+      - type
+      - url
+      - urlType
     TimelineAddEntries:
       properties:
         entries:
@@ -76,6 +156,44 @@ components:
           $ref: '#/components/schemas/InstructionType'
       required:
       - type
+    TimelineCoverBehavior:
+      properties:
+        type:
+          enum:
+          - TimelineCoverBehaviorDismiss
+          type: string
+      required:
+      - type
+    TimelineHalfCover:
+      properties:
+        dismissible:
+          type: boolean
+        halfCoverDisplayType:
+          enum:
+          - Cover
+          type: string
+        impressionCallbacks:
+          items:
+            $ref: '#/components/schemas/Callback'
+          type: array
+        primaryCoverCta:
+          $ref: '#/components/schemas/CoverCta'
+        primaryText:
+          $ref: '#/components/schemas/Text'
+        secondaryText:
+          $ref: '#/components/schemas/Text'
+        type:
+          enum:
+          - TimelineHalfCover
+          type: string
+      required:
+      - type
+      - halfCoverDisplayType
+      - primaryText
+      - primaryCoverCta
+      - secondaryText
+      - impressionCallbacks
+      - dismissible
     TimelinePinEntry:
       properties:
         entry:
@@ -137,6 +255,18 @@ components:
       - type
       - usersResults
       - richText
+    TimelineShowCover:
+      properties:
+        clientEventInfo:
+          $ref: ./content.yaml#/components/schemas/ClientEventInfo
+        cover:
+          $ref: '#/components/schemas/TimelineHalfCover'
+        type:
+          $ref: '#/components/schemas/InstructionType'
+      required:
+      - type
+      - clientEventInfo
+      - cover
     TimelineTerminateTimeline:
       properties:
         direction:

dist/compatible/schemas/tweet.yaml

@@ -269,11 +269,8 @@ components:
           $ref: '#/components/schemas/TweetView'
       required:
       - rest_id
-      - core
       - edit_control
       - is_translatable
-      - source
-      - legacy
       - views
     TweetCard:
       properties:
@@ -498,8 +495,6 @@ components:
           enum:
           - Available
           type: string
-      required:
-      - status
     mediaStats:
       properties:
         viewCount:

dist/docs/schemas/content.yaml

@@ -1,5 +1,14 @@
 components:
   schemas:
+    ClientEventInfo:
+      properties:
+        component:
+          type: string
+        details:
+          additionalProperties: true
+          type: object
+        element:
+          type: string
     ContentEntryType:
       enum:
       - TimelineTimelineItem
@@ -54,13 +63,11 @@ components:
           $ref: ./typename.yaml#/components/schemas/TypeName
         result:
           $ref: ./tweet.yaml#/components/schemas/TweetUnion
-      required:
+      required: null
-      - result
     ModuleEntry:
       properties:
         clientEventInfo:
-          additionalProperties: true
+          $ref: '#/components/schemas/ClientEventInfo'
-          type: object
         itemContent:
           $ref: '#/components/schemas/ItemContentUnion'
       required:
@@ -116,8 +123,7 @@ components:
         __typename:
           $ref: ./typename.yaml#/components/schemas/TypeName
         clientEventInfo:
-          additionalProperties: true
+          $ref: '#/components/schemas/ClientEventInfo'
-          type: object
         entryType:
           $ref: '#/components/schemas/ContentEntryType'
         feedbackInfo:

dist/docs/schemas/instruction.yaml

@@ -1,5 +1,42 @@
 components:
   schemas:
+    Callback:
+      properties:
+        endpoint:
+          format: uri
+          type: string
+      required:
+      - endpoint
+    CoverCta:
+      properties:
+        Text:
+          type: string
+        buttonStyle:
+          enum:
+          - Primary
+          type: string
+        callbacks:
+          items:
+            $ref: '#/components/schemas/Callback'
+          type: array
+        clientEventInfo:
+          $ref: '#/components/schemas/CtaClientEventInfo'
+        ctaBehavior:
+          $ref: '#/components/schemas/TimelineCoverBehavior'
+      required:
+      - text
+      - ctaBehavior
+      - callbacks
+      - clientEventInfo
+      - buttonStyle
+    CtaClientEventInfo:
+      properties:
+        action:
+          enum:
+          - primary_cta
+          type: string
+      required:
+      - action
     InstructionType:
       enum:
       - TimelineAddEntries
@@ -9,6 +46,7 @@ components:
       - TimelineReplaceEntry
       - TimelineShowAlert
       - TimelineTerminateTimeline
+      - TimelineShowCover
       type: string
     InstructionUnion:
       discriminator:
@@ -19,6 +57,7 @@ components:
           TimelinePinEntry: '#/components/schemas/TimelinePinEntry'
           TimelineReplaceEntry: '#/components/schemas/TimelineReplaceEntry'
           TimelineShowAlert: '#/components/schemas/TimelineShowAlert'
+          TimelineShowCover: '#/components/schemas/TimelineShowCover'
           TimelineTerminateTimeline: '#/components/schemas/TimelineTerminateTimeline'
         propertyName: type
       oneOf:
@@ -29,6 +68,47 @@ components:
       - $ref: '#/components/schemas/TimelineReplaceEntry'
       - $ref: '#/components/schemas/TimelineShowAlert'
       - $ref: '#/components/schemas/TimelineTerminateTimeline'
+      - $ref: '#/components/schemas/TimelineShowCover'
+    Text:
+      properties:
+        entities:
+          items:
+            $ref: '#/components/schemas/TextEntity'
+          type: array
+        text:
+          type: string
+      required:
+      - text
+      - entities
+    TextEntity:
+      properties:
+        fromIndex:
+          type: integer
+        ref:
+          $ref: '#/components/schemas/TextEntityRef'
+        toIndex:
+          type: integer
+      required:
+      - fromIndex
+      - toIndex
+      - ref
+    TextEntityRef:
+      properties:
+        type:
+          enum:
+          - TimelineUrl
+          type: string
+        url:
+          format: uri
+          type: string
+        urlType:
+          enum:
+          - ExternalUrl
+          type: string
+      required:
+      - type
+      - url
+      - urlType
     TimelineAddEntries:
       properties:
         entries:
@@ -76,6 +156,44 @@ components:
           $ref: '#/components/schemas/InstructionType'
       required:
       - type
+    TimelineCoverBehavior:
+      properties:
+        type:
+          enum:
+          - TimelineCoverBehaviorDismiss
+          type: string
+      required:
+      - type
+    TimelineHalfCover:
+      properties:
+        dismissible:
+          type: boolean
+        halfCoverDisplayType:
+          enum:
+          - Cover
+          type: string
+        impressionCallbacks:
+          items:
+            $ref: '#/components/schemas/Callback'
+          type: array
+        primaryCoverCta:
+          $ref: '#/components/schemas/CoverCta'
+        primaryText:
+          $ref: '#/components/schemas/Text'
+        secondaryText:
+          $ref: '#/components/schemas/Text'
+        type:
+          enum:
+          - TimelineHalfCover
+          type: string
+      required:
+      - type
+      - halfCoverDisplayType
+      - primaryText
+      - primaryCoverCta
+      - secondaryText
+      - impressionCallbacks
+      - dismissible
     TimelinePinEntry:
       properties:
         entry:
@@ -137,6 +255,18 @@ components:
       - type
       - usersResults
       - richText
+    TimelineShowCover:
+      properties:
+        clientEventInfo:
+          $ref: ./content.yaml#/components/schemas/ClientEventInfo
+        cover:
+          $ref: '#/components/schemas/TimelineHalfCover'
+        type:
+          $ref: '#/components/schemas/InstructionType'
+      required:
+      - type
+      - clientEventInfo
+      - cover
     TimelineTerminateTimeline:
       properties:
         direction:

dist/docs/schemas/tweet.yaml

@@ -269,11 +269,8 @@ components:
           $ref: '#/components/schemas/TweetView'
       required:
       - rest_id
-      - core
       - edit_control
       - is_translatable
-      - source
-      - legacy
       - views
     TweetCard:
       properties:
@@ -498,8 +495,6 @@ components:
           enum:
           - Available
           type: string
-      required:
-      - status
     mediaStats:
       properties:
         viewCount:

src/openapi/schemas/content.yaml

@@ -40,8 +40,7 @@ components:
         itemContent:
           $ref: "#/components/schemas/ItemContentUnion"
         clientEventInfo:
-          type: object
+          $ref: "#/components/schemas/ClientEventInfo"
-          additionalProperties: true # todo
         feedbackInfo:
           type: object
           additionalProperties: true # todo
@@ -113,8 +112,7 @@ components:
         - "itemContent"
       properties:
         clientEventInfo:
-          type: object
+          $ref: "#/components/schemas/ClientEventInfo"
-          additionalProperties: true # todo
         itemContent:
           $ref: "#/components/schemas/ItemContentUnion"
 
@@ -191,7 +189,6 @@ components:
 
     ItemResult:
       required:
-        - "result"
       properties:
         __typename:
           $ref: "./typename.yaml#/components/schemas/TypeName" # null | TimelineTweet
@@ -219,3 +216,17 @@ components:
         __typename:
           $ref: "./typename.yaml#/components/schemas/TypeName" # TimelineMessagePrompt
       additionalProperties: true # todo
+
+    ClientEventInfo:
+      properties:
+        component:
+          # enum half_cover
+          type: string
+        element:
+          type: string
+          # august-2023-privacy-prompt-candidate
+          # pattern: "(([a-z]+|[0-9]+|[0-9a-f]+)(-|$))+"
+          # pattern: "^(january|february|march|april|may|june|july|august|september|october|november|december)-[0-9]{4}-([a-z]-)+[a-z]+$"
+        details:
+          type: object
+          additionalProperties: true # todo

src/openapi/schemas/instruction.yaml

@@ -15,6 +15,8 @@ components:
         - $ref: "#/components/schemas/TimelineReplaceEntry"
         - $ref: "#/components/schemas/TimelineShowAlert"
         - $ref: "#/components/schemas/TimelineTerminateTimeline"
+        - $ref: "#/components/schemas/TimelineShowCover"
+
       discriminator:
         propertyName: type
         mapping": # deprecated
@@ -25,6 +27,7 @@ components:
           TimelineReplaceEntry: "#/components/schemas/TimelineReplaceEntry"
           TimelineShowAlert: "#/components/schemas/TimelineShowAlert"
           TimelineTerminateTimeline: "#/components/schemas/TimelineTerminateTimeline"
+          TimelineShowCover: "#/components/schemas/TimelineShowCover"
 
     InstructionType:
       type: string
@@ -37,6 +40,7 @@ components:
           TimelineReplaceEntry,
           TimelineShowAlert,
           TimelineTerminateTimeline,
+          TimelineShowCover,
         ]
 
     TimelineAddEntries:
@@ -162,3 +166,133 @@ components:
         sortIndex:
           type: string
           pattern: "[0-9]+$"
+
+    TimelineShowCover:
+      required:
+        - type
+        - clientEventInfo
+        - cover
+      properties:
+        type:
+          $ref: "#/components/schemas/InstructionType" # TimelineShowCover
+        clientEventInfo:
+          $ref: "./content.yaml#/components/schemas/ClientEventInfo"
+        cover:
+          $ref: "#/components/schemas/TimelineHalfCover"
+
+    TimelineHalfCover:
+      required:
+        - type
+        - halfCoverDisplayType
+        - primaryText
+        - primaryCoverCta
+        - secondaryText
+        - impressionCallbacks
+        - dismissible
+      properties:
+        type:
+          type: string
+          enum: [TimelineHalfCover]
+        halfCoverDisplayType:
+          type: string
+          enum: [Cover]
+        primaryText:
+          $ref: "#/components/schemas/Text"
+        secondaryText:
+          $ref: "#/components/schemas/Text"
+        primaryCoverCta:
+          $ref: "#/components/schemas/CoverCta"
+        impressionCallbacks:
+          type: array
+          items:
+            $ref: "#/components/schemas/Callback"
+        dismissible:
+          type: boolean
+
+    Text:
+      required:
+        - text
+        - entities
+      properties:
+        text:
+          type: string
+        entities:
+          type: array
+          items:
+            $ref: "#/components/schemas/TextEntity"
+
+    TextEntity:
+      required:
+        - fromIndex
+        - toIndex
+        - ref
+      properties:
+        fromIndex:
+          type: integer
+        toIndex:
+          type: integer
+        ref:
+          $ref: "#/components/schemas/TextEntityRef"
+
+    TextEntityRef:
+      required:
+        - type
+        - url
+        - urlType
+      properties:
+        type:
+          type: string
+          enum: [TimelineUrl]
+        url:
+          type: string
+          format: uri
+        urlType:
+          type: string
+          enum: [ExternalUrl]
+
+    CoverCta:
+      required:
+        - text
+        - ctaBehavior
+        - callbacks
+        - clientEventInfo
+        - buttonStyle
+      properties:
+        Text:
+          type: string
+        ctaBehavior:
+          $ref: "#/components/schemas/TimelineCoverBehavior"
+        callbacks:
+          type: array
+          items:
+            $ref: "#/components/schemas/Callback"
+        clientEventInfo:
+          $ref: "#/components/schemas/CtaClientEventInfo"
+        buttonStyle:
+          type: string
+          enum: ["Primary"]
+
+    TimelineCoverBehavior:
+      required:
+        - type
+      properties:
+        type:
+          type: string
+          enum: [TimelineCoverBehaviorDismiss]
+
+    Callback:
+      required:
+        - endpoint
+      properties:
+        endpoint:
+          type: string
+          format: uri
+          # pattern: '^/1\.1/[a-z]+/[a-z]+\.json\?[a-z_]+=[a-z0-9-]+(&[a-z_]+=[a-z0-9-]+)+?$' #/1.1/onboarding/fatigue.json?{params}
+
+    CtaClientEventInfo:
+      required:
+        - action
+      properties:
+        action:
+          type: string
+          enum: [primary_cta]

src/openapi/schemas/tweet.yaml

@@ -38,11 +38,12 @@ components:
     Tweet:
       required:
         - "rest_id"
-        - "core"
+        # If the tweet has been edited, this property does not exist.
+        # - "core"
         - "edit_control"
         - "is_translatable"
-        - "source"
+        # - "source"
-        - "legacy"
+        # - "legacy"
         - "views"
 
       properties:
@@ -496,8 +497,8 @@ components:
           type: integer
 
     extMediaAvailability:
-      required:
+      # required:
-        - "status"
+      #   - "status"
       properties:
         status:
           type: string

test/python/test_serialize.py

@@ -8,6 +8,7 @@ import time
 import glob
 import aenum
 import concurrent.futures
+import traceback
 
 
 logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s")
@@ -106,21 +107,23 @@ def save_cache(data):
         json.dump(data, f, indent=4)
 
 
-def task_callback(file):
+def task_callback(file, thread=True):
     try:
         with open(file, "r") as f:
             cache = json.load(f)
         data = pt.__dict__[cache["type"]].from_json(cache["raw"])
         rate = match_rate(data.to_dict(), json.loads(cache["raw"]))
-        return f"Match rate: {rate}"
+        return rate, file
     except Exception as e:
-        return f"Error: {e} {file}"
+        if thread:
+            return 0, file
+        else:
+            raise
 
 
 def error_dump(e):
     if ERROR_UNCATCHED:
         raise
-    import traceback
 
     logger.error("==========[STACK TRACE]==========")
     for trace in traceback.format_exc().split("\n"):
@@ -146,12 +149,20 @@ if __name__ == "__main__":
     with open("src/config/placeholder.json", "r") as f:
         placeholder = json.load(f)
 
+    fail = []
     with concurrent.futures.ProcessPoolExecutor() as executor:
         tasks = [executor.submit(task_callback, x) for x in glob.glob("cache/*.json")]
         for task in concurrent.futures.as_completed(tasks):
-            logger.info(task.result())
+            rate, file = task.result()
+            if rate < 1:
+                fail.append(file)
+            logger.info(f"Match rate: {rate}")
 
-    logger.info(f"Success: {len(glob.glob('cache/*.json'))}")
+    logger.info(f"Fail: {len(fail)} / {len(glob.glob('cache/*.json'))}")
+
+    for file in fail:
+        task_callback(file, thread=False)
+        logger.info(f"Match rate: {rate}")
 
     api_conf = pt.Configuration(
         api_key={