diff --git a/dist/compatible/schemas/content.yaml b/dist/compatible/schemas/content.yaml index ccadf95..46cfeb0 100644 --- a/dist/compatible/schemas/content.yaml +++ b/dist/compatible/schemas/content.yaml @@ -1,5 +1,14 @@ components: schemas: + ClientEventInfo: + properties: + component: + type: string + details: + additionalProperties: true + type: object + element: + type: string ContentEntryType: enum: - TimelineTimelineItem @@ -54,13 +63,11 @@ components: $ref: ./typename.yaml#/components/schemas/TypeName result: $ref: ./tweet.yaml#/components/schemas/TweetUnion - required: - - result + required: null ModuleEntry: properties: clientEventInfo: - additionalProperties: true - type: object + $ref: '#/components/schemas/ClientEventInfo' itemContent: $ref: '#/components/schemas/ItemContentUnion' required: @@ -116,8 +123,7 @@ components: __typename: $ref: ./typename.yaml#/components/schemas/TypeName clientEventInfo: - additionalProperties: true - type: object + $ref: '#/components/schemas/ClientEventInfo' entryType: $ref: '#/components/schemas/ContentEntryType' feedbackInfo: diff --git a/dist/compatible/schemas/instruction.yaml b/dist/compatible/schemas/instruction.yaml index 90a112a..5269f21 100644 --- a/dist/compatible/schemas/instruction.yaml +++ b/dist/compatible/schemas/instruction.yaml @@ -1,5 +1,42 @@ components: schemas: + Callback: + properties: + endpoint: + format: uri + type: string + required: + - endpoint + CoverCta: + properties: + Text: + type: string + buttonStyle: + enum: + - Primary + type: string + callbacks: + items: + $ref: '#/components/schemas/Callback' + type: array + clientEventInfo: + $ref: '#/components/schemas/CtaClientEventInfo' + ctaBehavior: + $ref: '#/components/schemas/TimelineCoverBehavior' + required: + - text + - ctaBehavior + - callbacks + - clientEventInfo + - buttonStyle + CtaClientEventInfo: + properties: + action: + enum: + - primary_cta + type: string + required: + - action InstructionType: enum: - TimelineAddEntries @@ -9,6 +46,7 @@ components: - TimelineReplaceEntry - TimelineShowAlert - TimelineTerminateTimeline + - TimelineShowCover type: string InstructionUnion: discriminator: @@ -19,6 +57,7 @@ components: TimelinePinEntry: '#/components/schemas/TimelinePinEntry' TimelineReplaceEntry: '#/components/schemas/TimelineReplaceEntry' TimelineShowAlert: '#/components/schemas/TimelineShowAlert' + TimelineShowCover: '#/components/schemas/TimelineShowCover' TimelineTerminateTimeline: '#/components/schemas/TimelineTerminateTimeline' propertyName: type oneOf: @@ -29,6 +68,47 @@ components: - $ref: '#/components/schemas/TimelineReplaceEntry' - $ref: '#/components/schemas/TimelineShowAlert' - $ref: '#/components/schemas/TimelineTerminateTimeline' + - $ref: '#/components/schemas/TimelineShowCover' + Text: + properties: + entities: + items: + $ref: '#/components/schemas/TextEntity' + type: array + text: + type: string + required: + - text + - entities + TextEntity: + properties: + fromIndex: + type: integer + ref: + $ref: '#/components/schemas/TextEntityRef' + toIndex: + type: integer + required: + - fromIndex + - toIndex + - ref + TextEntityRef: + properties: + type: + enum: + - TimelineUrl + type: string + url: + format: uri + type: string + urlType: + enum: + - ExternalUrl + type: string + required: + - type + - url + - urlType TimelineAddEntries: properties: entries: @@ -76,6 +156,44 @@ components: $ref: '#/components/schemas/InstructionType' required: - type + TimelineCoverBehavior: + properties: + type: + enum: + - TimelineCoverBehaviorDismiss + type: string + required: + - type + TimelineHalfCover: + properties: + dismissible: + type: boolean + halfCoverDisplayType: + enum: + - Cover + type: string + impressionCallbacks: + items: + $ref: '#/components/schemas/Callback' + type: array + primaryCoverCta: + $ref: '#/components/schemas/CoverCta' + primaryText: + $ref: '#/components/schemas/Text' + secondaryText: + $ref: '#/components/schemas/Text' + type: + enum: + - TimelineHalfCover + type: string + required: + - type + - halfCoverDisplayType + - primaryText + - primaryCoverCta + - secondaryText + - impressionCallbacks + - dismissible TimelinePinEntry: properties: entry: @@ -137,6 +255,18 @@ components: - type - usersResults - richText + TimelineShowCover: + properties: + clientEventInfo: + $ref: ./content.yaml#/components/schemas/ClientEventInfo + cover: + $ref: '#/components/schemas/TimelineHalfCover' + type: + $ref: '#/components/schemas/InstructionType' + required: + - type + - clientEventInfo + - cover TimelineTerminateTimeline: properties: direction: diff --git a/dist/compatible/schemas/tweet.yaml b/dist/compatible/schemas/tweet.yaml index 454ee82..77c9fa7 100644 --- a/dist/compatible/schemas/tweet.yaml +++ b/dist/compatible/schemas/tweet.yaml @@ -269,11 +269,8 @@ components: $ref: '#/components/schemas/TweetView' required: - rest_id - - core - edit_control - is_translatable - - source - - legacy - views TweetCard: properties: @@ -498,8 +495,6 @@ components: enum: - Available type: string - required: - - status mediaStats: properties: viewCount: diff --git a/dist/docs/schemas/content.yaml b/dist/docs/schemas/content.yaml index ccadf95..46cfeb0 100644 --- a/dist/docs/schemas/content.yaml +++ b/dist/docs/schemas/content.yaml @@ -1,5 +1,14 @@ components: schemas: + ClientEventInfo: + properties: + component: + type: string + details: + additionalProperties: true + type: object + element: + type: string ContentEntryType: enum: - TimelineTimelineItem @@ -54,13 +63,11 @@ components: $ref: ./typename.yaml#/components/schemas/TypeName result: $ref: ./tweet.yaml#/components/schemas/TweetUnion - required: - - result + required: null ModuleEntry: properties: clientEventInfo: - additionalProperties: true - type: object + $ref: '#/components/schemas/ClientEventInfo' itemContent: $ref: '#/components/schemas/ItemContentUnion' required: @@ -116,8 +123,7 @@ components: __typename: $ref: ./typename.yaml#/components/schemas/TypeName clientEventInfo: - additionalProperties: true - type: object + $ref: '#/components/schemas/ClientEventInfo' entryType: $ref: '#/components/schemas/ContentEntryType' feedbackInfo: diff --git a/dist/docs/schemas/instruction.yaml b/dist/docs/schemas/instruction.yaml index 90a112a..5269f21 100644 --- a/dist/docs/schemas/instruction.yaml +++ b/dist/docs/schemas/instruction.yaml @@ -1,5 +1,42 @@ components: schemas: + Callback: + properties: + endpoint: + format: uri + type: string + required: + - endpoint + CoverCta: + properties: + Text: + type: string + buttonStyle: + enum: + - Primary + type: string + callbacks: + items: + $ref: '#/components/schemas/Callback' + type: array + clientEventInfo: + $ref: '#/components/schemas/CtaClientEventInfo' + ctaBehavior: + $ref: '#/components/schemas/TimelineCoverBehavior' + required: + - text + - ctaBehavior + - callbacks + - clientEventInfo + - buttonStyle + CtaClientEventInfo: + properties: + action: + enum: + - primary_cta + type: string + required: + - action InstructionType: enum: - TimelineAddEntries @@ -9,6 +46,7 @@ components: - TimelineReplaceEntry - TimelineShowAlert - TimelineTerminateTimeline + - TimelineShowCover type: string InstructionUnion: discriminator: @@ -19,6 +57,7 @@ components: TimelinePinEntry: '#/components/schemas/TimelinePinEntry' TimelineReplaceEntry: '#/components/schemas/TimelineReplaceEntry' TimelineShowAlert: '#/components/schemas/TimelineShowAlert' + TimelineShowCover: '#/components/schemas/TimelineShowCover' TimelineTerminateTimeline: '#/components/schemas/TimelineTerminateTimeline' propertyName: type oneOf: @@ -29,6 +68,47 @@ components: - $ref: '#/components/schemas/TimelineReplaceEntry' - $ref: '#/components/schemas/TimelineShowAlert' - $ref: '#/components/schemas/TimelineTerminateTimeline' + - $ref: '#/components/schemas/TimelineShowCover' + Text: + properties: + entities: + items: + $ref: '#/components/schemas/TextEntity' + type: array + text: + type: string + required: + - text + - entities + TextEntity: + properties: + fromIndex: + type: integer + ref: + $ref: '#/components/schemas/TextEntityRef' + toIndex: + type: integer + required: + - fromIndex + - toIndex + - ref + TextEntityRef: + properties: + type: + enum: + - TimelineUrl + type: string + url: + format: uri + type: string + urlType: + enum: + - ExternalUrl + type: string + required: + - type + - url + - urlType TimelineAddEntries: properties: entries: @@ -76,6 +156,44 @@ components: $ref: '#/components/schemas/InstructionType' required: - type + TimelineCoverBehavior: + properties: + type: + enum: + - TimelineCoverBehaviorDismiss + type: string + required: + - type + TimelineHalfCover: + properties: + dismissible: + type: boolean + halfCoverDisplayType: + enum: + - Cover + type: string + impressionCallbacks: + items: + $ref: '#/components/schemas/Callback' + type: array + primaryCoverCta: + $ref: '#/components/schemas/CoverCta' + primaryText: + $ref: '#/components/schemas/Text' + secondaryText: + $ref: '#/components/schemas/Text' + type: + enum: + - TimelineHalfCover + type: string + required: + - type + - halfCoverDisplayType + - primaryText + - primaryCoverCta + - secondaryText + - impressionCallbacks + - dismissible TimelinePinEntry: properties: entry: @@ -137,6 +255,18 @@ components: - type - usersResults - richText + TimelineShowCover: + properties: + clientEventInfo: + $ref: ./content.yaml#/components/schemas/ClientEventInfo + cover: + $ref: '#/components/schemas/TimelineHalfCover' + type: + $ref: '#/components/schemas/InstructionType' + required: + - type + - clientEventInfo + - cover TimelineTerminateTimeline: properties: direction: diff --git a/dist/docs/schemas/tweet.yaml b/dist/docs/schemas/tweet.yaml index 454ee82..77c9fa7 100644 --- a/dist/docs/schemas/tweet.yaml +++ b/dist/docs/schemas/tweet.yaml @@ -269,11 +269,8 @@ components: $ref: '#/components/schemas/TweetView' required: - rest_id - - core - edit_control - is_translatable - - source - - legacy - views TweetCard: properties: @@ -498,8 +495,6 @@ components: enum: - Available type: string - required: - - status mediaStats: properties: viewCount: diff --git a/src/openapi/schemas/content.yaml b/src/openapi/schemas/content.yaml index 95508a9..aa2e511 100644 --- a/src/openapi/schemas/content.yaml +++ b/src/openapi/schemas/content.yaml @@ -40,8 +40,7 @@ components: itemContent: $ref: "#/components/schemas/ItemContentUnion" clientEventInfo: - type: object - additionalProperties: true # todo + $ref: "#/components/schemas/ClientEventInfo" feedbackInfo: type: object additionalProperties: true # todo @@ -113,8 +112,7 @@ components: - "itemContent" properties: clientEventInfo: - type: object - additionalProperties: true # todo + $ref: "#/components/schemas/ClientEventInfo" itemContent: $ref: "#/components/schemas/ItemContentUnion" @@ -191,7 +189,6 @@ components: ItemResult: required: - - "result" properties: __typename: $ref: "./typename.yaml#/components/schemas/TypeName" # null | TimelineTweet @@ -219,3 +216,17 @@ components: __typename: $ref: "./typename.yaml#/components/schemas/TypeName" # TimelineMessagePrompt additionalProperties: true # todo + + ClientEventInfo: + properties: + component: + # enum half_cover + type: string + element: + type: string + # august-2023-privacy-prompt-candidate + # pattern: "(([a-z]+|[0-9]+|[0-9a-f]+)(-|$))+" + # pattern: "^(january|february|march|april|may|june|july|august|september|october|november|december)-[0-9]{4}-([a-z]-)+[a-z]+$" + details: + type: object + additionalProperties: true # todo diff --git a/src/openapi/schemas/instruction.yaml b/src/openapi/schemas/instruction.yaml index 881f9dd..25a3def 100644 --- a/src/openapi/schemas/instruction.yaml +++ b/src/openapi/schemas/instruction.yaml @@ -15,6 +15,8 @@ components: - $ref: "#/components/schemas/TimelineReplaceEntry" - $ref: "#/components/schemas/TimelineShowAlert" - $ref: "#/components/schemas/TimelineTerminateTimeline" + - $ref: "#/components/schemas/TimelineShowCover" + discriminator: propertyName: type mapping": # deprecated @@ -25,6 +27,7 @@ components: TimelineReplaceEntry: "#/components/schemas/TimelineReplaceEntry" TimelineShowAlert: "#/components/schemas/TimelineShowAlert" TimelineTerminateTimeline: "#/components/schemas/TimelineTerminateTimeline" + TimelineShowCover: "#/components/schemas/TimelineShowCover" InstructionType: type: string @@ -37,6 +40,7 @@ components: TimelineReplaceEntry, TimelineShowAlert, TimelineTerminateTimeline, + TimelineShowCover, ] TimelineAddEntries: @@ -162,3 +166,133 @@ components: sortIndex: type: string pattern: "[0-9]+$" + + TimelineShowCover: + required: + - type + - clientEventInfo + - cover + properties: + type: + $ref: "#/components/schemas/InstructionType" # TimelineShowCover + clientEventInfo: + $ref: "./content.yaml#/components/schemas/ClientEventInfo" + cover: + $ref: "#/components/schemas/TimelineHalfCover" + + TimelineHalfCover: + required: + - type + - halfCoverDisplayType + - primaryText + - primaryCoverCta + - secondaryText + - impressionCallbacks + - dismissible + properties: + type: + type: string + enum: [TimelineHalfCover] + halfCoverDisplayType: + type: string + enum: [Cover] + primaryText: + $ref: "#/components/schemas/Text" + secondaryText: + $ref: "#/components/schemas/Text" + primaryCoverCta: + $ref: "#/components/schemas/CoverCta" + impressionCallbacks: + type: array + items: + $ref: "#/components/schemas/Callback" + dismissible: + type: boolean + + Text: + required: + - text + - entities + properties: + text: + type: string + entities: + type: array + items: + $ref: "#/components/schemas/TextEntity" + + TextEntity: + required: + - fromIndex + - toIndex + - ref + properties: + fromIndex: + type: integer + toIndex: + type: integer + ref: + $ref: "#/components/schemas/TextEntityRef" + + TextEntityRef: + required: + - type + - url + - urlType + properties: + type: + type: string + enum: [TimelineUrl] + url: + type: string + format: uri + urlType: + type: string + enum: [ExternalUrl] + + CoverCta: + required: + - text + - ctaBehavior + - callbacks + - clientEventInfo + - buttonStyle + properties: + Text: + type: string + ctaBehavior: + $ref: "#/components/schemas/TimelineCoverBehavior" + callbacks: + type: array + items: + $ref: "#/components/schemas/Callback" + clientEventInfo: + $ref: "#/components/schemas/CtaClientEventInfo" + buttonStyle: + type: string + enum: ["Primary"] + + TimelineCoverBehavior: + required: + - type + properties: + type: + type: string + enum: [TimelineCoverBehaviorDismiss] + + Callback: + required: + - endpoint + properties: + endpoint: + type: string + format: uri + # pattern: '^/1\.1/[a-z]+/[a-z]+\.json\?[a-z_]+=[a-z0-9-]+(&[a-z_]+=[a-z0-9-]+)+?$' #/1.1/onboarding/fatigue.json?{params} + + CtaClientEventInfo: + required: + - action + properties: + action: + type: string + enum: [primary_cta] diff --git a/src/openapi/schemas/tweet.yaml b/src/openapi/schemas/tweet.yaml index 1fafda0..5ae992f 100644 --- a/src/openapi/schemas/tweet.yaml +++ b/src/openapi/schemas/tweet.yaml @@ -38,11 +38,12 @@ components: Tweet: required: - "rest_id" - - "core" + # If the tweet has been edited, this property does not exist. + # - "core" - "edit_control" - "is_translatable" - - "source" - - "legacy" + # - "source" + # - "legacy" - "views" properties: @@ -496,8 +497,8 @@ components: type: integer extMediaAvailability: - required: - - "status" + # required: + # - "status" properties: status: type: string diff --git a/test/python/test_serialize.py b/test/python/test_serialize.py index bd7f518..4968856 100644 --- a/test/python/test_serialize.py +++ b/test/python/test_serialize.py @@ -8,6 +8,7 @@ import time import glob import aenum import concurrent.futures +import traceback logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s") @@ -106,21 +107,23 @@ def save_cache(data): json.dump(data, f, indent=4) -def task_callback(file): +def task_callback(file, thread=True): try: with open(file, "r") as f: cache = json.load(f) data = pt.__dict__[cache["type"]].from_json(cache["raw"]) rate = match_rate(data.to_dict(), json.loads(cache["raw"])) - return f"Match rate: {rate}" + return rate, file except Exception as e: - return f"Error: {e} {file}" + if thread: + return 0, file + else: + raise def error_dump(e): if ERROR_UNCATCHED: raise - import traceback logger.error("==========[STACK TRACE]==========") for trace in traceback.format_exc().split("\n"): @@ -146,12 +149,20 @@ if __name__ == "__main__": with open("src/config/placeholder.json", "r") as f: placeholder = json.load(f) + fail = [] with concurrent.futures.ProcessPoolExecutor() as executor: tasks = [executor.submit(task_callback, x) for x in glob.glob("cache/*.json")] for task in concurrent.futures.as_completed(tasks): - logger.info(task.result()) + rate, file = task.result() + if rate < 1: + fail.append(file) + logger.info(f"Match rate: {rate}") - logger.info(f"Success: {len(glob.glob('cache/*.json'))}") + logger.info(f"Fail: {len(fail)} / {len(glob.glob('cache/*.json'))}") + + for file in fail: + task_callback(file, thread=False) + logger.info(f"Match rate: {rate}") api_conf = pt.Configuration( api_key={