From f15b0641234cbf2f8a40af3bc703a80959b0c9be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 02:29:37 +0900 Subject: [PATCH 01/24] update task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- .vscode/tasks.json | 3 +++ test/python/replace.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 test/python/replace.py diff --git a/.vscode/tasks.json b/.vscode/tasks.json index e7dff9e..3e8680f 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -6,8 +6,10 @@ "type": "shell", "linux": { "command": [ + "source .venv/bin/activate;", "python3 tools/build.py;", "java -jar openapi-generator-cli.jar generate -c test/python/openapi-generator-config.yaml -g python;", + "python3 test/python/replace.py;", "python3 -m pip install ./python_generated;" ] }, @@ -16,6 +18,7 @@ ".venv/Scripts/activate;", "python tools/build.py;", "java -jar openapi-generator-cli.jar generate -c test/python/openapi-generator-config.yaml -g python;", + "python test/python/replace.py;", "python -m pip install ./python_generated;" ] } diff --git a/test/python/replace.py b/test/python/replace.py new file mode 100644 index 0000000..ab9b5f9 --- /dev/null +++ b/test/python/replace.py @@ -0,0 +1,23 @@ +import glob + +for file in glob.glob("python_generated/openapi_client/models/*.py"): + with open(file, "r") as f: + text = f.read() + + indent = " " + + text = text.replace( + f"{indent}{indent}try:", + f"{indent}{indent}if match == 0:", + ) + text = text.replace( + f"{indent}{indent}except (ValidationError, ValueError) as e:", + f"{indent}{indent}else:", + ) + text = text.replace( + f"{indent}{indent}{indent}error_messages.append(str(e))", + f"{indent}{indent}{indent}pass", + ) + + with open(file, "w") as f: + f.write(text) From ac69c0c92bfb88973aef88c4e6c9c27047fd8b97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 02:29:47 +0900 Subject: [PATCH 02/24] add #35 wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/tweet.yaml | 187 ++++++++++++++++++++++++++++----- 1 file changed, 163 insertions(+), 24 deletions(-) diff --git a/src/openapi/schemas/tweet.yaml b/src/openapi/schemas/tweet.yaml index baf63f8..9a13aee 100644 --- a/src/openapi/schemas/tweet.yaml +++ b/src/openapi/schemas/tweet.yaml @@ -218,7 +218,6 @@ components: - "symbols" - "user_mentions" - "urls" - # - "media" properties: hashtags: type: array @@ -271,27 +270,25 @@ components: items: type: integer - ExtendedEntities: - required: - - "media" - properties: - media: - type: array - items: - $ref: "#/components/schemas/Media" - Media: required: + - "display_url" + - "expanded_url" - "id_str" - "indices" - "media_url_https" - - "url" - - "display_url" - - "expanded_url" - "type" + - "url" + - "features" - "sizes" - "original_info" properties: + display_url: + type: string + format: uri + expanded_url: + type: string + format: uri id_str: type: string pattern: "^[0-9]+$" @@ -302,20 +299,13 @@ components: media_url_https: type: string format: uri + type: + type: string # enum photo url: type: string format: uri - display_url: - type: string - format: uri - expanded_url: - type: string - format: uri - type: - type: string # enum sizes: - type: object - additionalProperties: true # todo + $ref: "#/components/schemas/MediaSizes" original_info: type: object properties: @@ -328,9 +318,158 @@ components: items: type: object additionalProperties: true # todo + + ExtendedEntities: + required: + - "media" + properties: + media: + type: array + items: + $ref: "#/components/schemas/MediaExtended" + + MediaExtended: + required: + - "display_url" + - "expanded_url" + - "id_str" + - "indices" + - "media_key" + - "media_url_https" + - "type" + - "url" + - "ext_media_availability" + - "features" + - "sizes" + - "original_info" + properties: + display_url: + type: string + format: uri + expanded_url: + type: string + format: uri + id_str: + type: string + pattern: "^[0-9]+$" + indices: + type: array + items: + type: integer media_key: type: string - pattern: "^[0-9]+_[0-9]+$" + media_url_https: + type: string + format: uri + type: + type: string # enum # photo | video + url: + type: string + format: uri + additional_media_info: + type: object + additionalProperties: true # todo + mediaStats: + type: object + additionalProperties: true # todo ext_media_availability: type: object additionalProperties: true # todo + features: + type: object + sizes: + $ref: "#/components/schemas/MediaSizes" + original_info: + $ref: "#/components/schemas/MediaOriginalInfo" + video_info: + $ref: "#/components/schemas/MediaVideoInfo" + + MediaOriginalInfo: + required: + - "height" + - "width" + properties: + height: + type: integer + width: + type: integer + focus_rects: + type: array + items: + $ref: "#/components/schemas/MediaOriginalInfoFocusRect" + + MediaOriginalInfoFocusRect: + type: object + required: + - "x" + - "y" + - "w" + - "h" + properties: + x: + type: integer + y: + type: integer + w: + type: integer + h: + type: integer + + MediaVideoInfo: + required: + - "aspect_ratio" + - "variants" + properties: + aspect_ratio: + type: array + items: + type: integer + duration_millis: + type: integer + variants: + type: array + items: + $ref: "#/components/schemas/MediaVideoInfoVariant" + + MediaVideoInfoVariant: + required: + - "content_type" + - "url" + properties: + bitrate: + type: integer + content_type: + type: string + url: + type: string + format: uri + + MediaSizes: + required: + - "large" + - "medium" + - "small" + - "thumb" + properties: + large: + $ref: "#/components/schemas/MediaSize" + medium: + $ref: "#/components/schemas/MediaSize" + small: + $ref: "#/components/schemas/MediaSize" + thumb: + $ref: "#/components/schemas/MediaSize" + + MediaSize: + required: + - "w" + - "h" + - "resize" + properties: + w: + type: integer + h: + type: integer + resize: + type: string # enum fit + From 34bb3d07f3a9402bc88766036f1589f981a7c330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 02:29:52 +0900 Subject: [PATCH 03/24] build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- dist/compatible/schemas/tweet.yaml | 157 +++++++++++++++++++++++++++-- dist/docs/schemas/tweet.yaml | 157 +++++++++++++++++++++++++++-- 2 files changed, 292 insertions(+), 22 deletions(-) diff --git a/dist/compatible/schemas/tweet.yaml b/dist/compatible/schemas/tweet.yaml index bea1c3c..c027220 100644 --- a/dist/compatible/schemas/tweet.yaml +++ b/dist/compatible/schemas/tweet.yaml @@ -31,7 +31,7 @@ components: properties: media: items: - $ref: '#/components/schemas/Media' + $ref: '#/components/schemas/MediaExtended' type: array required: - media @@ -46,9 +46,6 @@ components: expanded_url: format: uri type: string - ext_media_availability: - additionalProperties: true - type: object id_str: pattern: ^[0-9]+$ type: string @@ -56,9 +53,6 @@ components: items: type: integer type: array - media_key: - pattern: ^[0-9]+_[0-9]+$ - type: string media_url_https: format: uri type: string @@ -75,6 +69,59 @@ components: type: integer type: object sizes: + $ref: '#/components/schemas/MediaSizes' + type: + type: string + url: + format: uri + type: string + required: + - display_url + - expanded_url + - id_str + - indices + - media_url_https + - type + - url + - features + - sizes + - original_info + MediaExtended: + properties: + additional_media_info: + additionalProperties: true + type: object + display_url: + format: uri + type: string + expanded_url: + format: uri + type: string + ext_media_availability: + additionalProperties: true + type: object + features: + type: object + id_str: + pattern: ^[0-9]+$ + type: string + indices: + items: + type: integer + type: array + mediaStats: + additionalProperties: true + type: object + media_key: + type: string + media_url_https: + format: uri + type: string + original_info: + $ref: '#/components/schemas/MediaOriginalInfo' + sizes: + $ref: '#/components/schemas/MediaSizes' + status: additionalProperties: true type: object type: @@ -82,16 +129,104 @@ components: url: format: uri type: string + video_info: + $ref: '#/components/schemas/MediaVideoInfo' required: - - id_str - - indices - - media_url_https - - url - display_url - expanded_url + - id_str + - indices + - media_key + - media_url_https - type + - url + - ext_media_availability + - features - sizes - original_info + MediaOriginalInfo: + properties: + focus_rects: + items: + $ref: '#/components/schemas/MediaOriginalInfoFocusRect' + type: array + height: + type: integer + width: + type: integer + required: + - height + - width + MediaOriginalInfoFocusRect: + properties: + h: + type: integer + w: + type: integer + x: + type: integer + y: + type: integer + required: + - x + - y + - w + - h + type: object + MediaSize: + properties: + h: + type: integer + resize: + type: string + w: + type: integer + required: + - w + - h + - resize + MediaSizes: + properties: + large: + $ref: '#/components/schemas/MediaSize' + medium: + $ref: '#/components/schemas/MediaSize' + small: + $ref: '#/components/schemas/MediaSize' + thumb: + $ref: '#/components/schemas/MediaSize' + required: + - large + - medium + - small + - thumb + MediaVideoInfo: + properties: + aspect_ratio: + items: + type: integer + type: array + duration_millis: + type: integer + variants: + items: + $ref: '#/components/schemas/MediaVideoInfoVariant' + type: array + required: + - aspect_ratio + - variants + MediaVideoInfoVariant: + properties: + bitrate: + type: integer + content_type: + type: string + url: + format: uri + type: string + required: + - content_type + - url Symbol: additionalProperties: true type: object diff --git a/dist/docs/schemas/tweet.yaml b/dist/docs/schemas/tweet.yaml index bea1c3c..c027220 100644 --- a/dist/docs/schemas/tweet.yaml +++ b/dist/docs/schemas/tweet.yaml @@ -31,7 +31,7 @@ components: properties: media: items: - $ref: '#/components/schemas/Media' + $ref: '#/components/schemas/MediaExtended' type: array required: - media @@ -46,9 +46,6 @@ components: expanded_url: format: uri type: string - ext_media_availability: - additionalProperties: true - type: object id_str: pattern: ^[0-9]+$ type: string @@ -56,9 +53,6 @@ components: items: type: integer type: array - media_key: - pattern: ^[0-9]+_[0-9]+$ - type: string media_url_https: format: uri type: string @@ -75,6 +69,59 @@ components: type: integer type: object sizes: + $ref: '#/components/schemas/MediaSizes' + type: + type: string + url: + format: uri + type: string + required: + - display_url + - expanded_url + - id_str + - indices + - media_url_https + - type + - url + - features + - sizes + - original_info + MediaExtended: + properties: + additional_media_info: + additionalProperties: true + type: object + display_url: + format: uri + type: string + expanded_url: + format: uri + type: string + ext_media_availability: + additionalProperties: true + type: object + features: + type: object + id_str: + pattern: ^[0-9]+$ + type: string + indices: + items: + type: integer + type: array + mediaStats: + additionalProperties: true + type: object + media_key: + type: string + media_url_https: + format: uri + type: string + original_info: + $ref: '#/components/schemas/MediaOriginalInfo' + sizes: + $ref: '#/components/schemas/MediaSizes' + status: additionalProperties: true type: object type: @@ -82,16 +129,104 @@ components: url: format: uri type: string + video_info: + $ref: '#/components/schemas/MediaVideoInfo' required: - - id_str - - indices - - media_url_https - - url - display_url - expanded_url + - id_str + - indices + - media_key + - media_url_https - type + - url + - ext_media_availability + - features - sizes - original_info + MediaOriginalInfo: + properties: + focus_rects: + items: + $ref: '#/components/schemas/MediaOriginalInfoFocusRect' + type: array + height: + type: integer + width: + type: integer + required: + - height + - width + MediaOriginalInfoFocusRect: + properties: + h: + type: integer + w: + type: integer + x: + type: integer + y: + type: integer + required: + - x + - y + - w + - h + type: object + MediaSize: + properties: + h: + type: integer + resize: + type: string + w: + type: integer + required: + - w + - h + - resize + MediaSizes: + properties: + large: + $ref: '#/components/schemas/MediaSize' + medium: + $ref: '#/components/schemas/MediaSize' + small: + $ref: '#/components/schemas/MediaSize' + thumb: + $ref: '#/components/schemas/MediaSize' + required: + - large + - medium + - small + - thumb + MediaVideoInfo: + properties: + aspect_ratio: + items: + type: integer + type: array + duration_millis: + type: integer + variants: + items: + $ref: '#/components/schemas/MediaVideoInfoVariant' + type: array + required: + - aspect_ratio + - variants + MediaVideoInfoVariant: + properties: + bitrate: + type: integer + content_type: + type: string + url: + format: uri + type: string + required: + - content_type + - url Symbol: additionalProperties: true type: object From 860a2c4029a09d5da3fb3b058f82dd64b392a018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 09:30:45 +0900 Subject: [PATCH 04/24] add features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/tweet.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/openapi/schemas/tweet.yaml b/src/openapi/schemas/tweet.yaml index 9a13aee..f846341 100644 --- a/src/openapi/schemas/tweet.yaml +++ b/src/openapi/schemas/tweet.yaml @@ -304,6 +304,8 @@ components: url: type: string format: uri + features: + type: object sizes: $ref: "#/components/schemas/MediaSizes" original_info: From 6e8a424b5bf9b21ebb214884459224f332cd9b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 11:55:33 +0900 Subject: [PATCH 05/24] update #35 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/tweet.yaml | 42 ++++++++++++++++++------------- src/openapi/schemas/typename.yaml | 1 + 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/openapi/schemas/tweet.yaml b/src/openapi/schemas/tweet.yaml index f846341..73dd612 100644 --- a/src/openapi/schemas/tweet.yaml +++ b/src/openapi/schemas/tweet.yaml @@ -309,17 +309,7 @@ components: sizes: $ref: "#/components/schemas/MediaSizes" original_info: - type: object - properties: - height: - type: integer - width: - type: integer - focus_rects: - type: array - items: - type: object - additionalProperties: true # todo + $ref: "#/components/schemas/MediaOriginalInfo" ExtendedEntities: required: @@ -369,14 +359,11 @@ components: type: string format: uri additional_media_info: - type: object - additionalProperties: true # todo + $ref: "#/components/schemas/AdditionalMediaInfo" mediaStats: - type: object - additionalProperties: true # todo + $ref: "#/components/schemas/mediaStats" ext_media_availability: - type: object - additionalProperties: true # todo + $ref: "#/components/schemas/extMediaAvailability" features: type: object sizes: @@ -446,6 +433,27 @@ components: type: string format: uri + AdditionalMediaInfo: + required: + - "monetizable" + properties: + monetizable: + type: boolean + + mediaStats: + required: + - "viewCount" + properties: + viewCount: + type: integer + + extMediaAvailability: + required: + - "status" + properties: + status: + type: string # enum Available + MediaSizes: required: - "large" diff --git a/src/openapi/schemas/typename.yaml b/src/openapi/schemas/typename.yaml index ff3f29e..19c79bb 100644 --- a/src/openapi/schemas/typename.yaml +++ b/src/openapi/schemas/typename.yaml @@ -18,6 +18,7 @@ components: TweetTombstone, TimelinePrompt, TimelineMessagePrompt, + TweetUnavailable, Tweet, User, ] From d1932aadd2f74a36354001590a15c5af8b5dddee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 11:56:47 +0900 Subject: [PATCH 06/24] build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- dist/compatible/schemas/tweet.yaml | 44 +++++++++++++++------------ dist/compatible/schemas/typename.yaml | 1 + dist/docs/schemas/tweet.yaml | 44 +++++++++++++++------------ dist/docs/schemas/typename.yaml | 1 + 4 files changed, 50 insertions(+), 40 deletions(-) diff --git a/dist/compatible/schemas/tweet.yaml b/dist/compatible/schemas/tweet.yaml index c027220..99ac345 100644 --- a/dist/compatible/schemas/tweet.yaml +++ b/dist/compatible/schemas/tweet.yaml @@ -1,5 +1,11 @@ components: schemas: + AdditionalMediaInfo: + properties: + monetizable: + type: boolean + required: + - monetizable Entities: properties: hashtags: @@ -46,6 +52,8 @@ components: expanded_url: format: uri type: string + features: + type: object id_str: pattern: ^[0-9]+$ type: string @@ -57,17 +65,7 @@ components: format: uri type: string original_info: - properties: - focus_rects: - items: - additionalProperties: true - type: object - type: array - height: - type: integer - width: - type: integer - type: object + $ref: '#/components/schemas/MediaOriginalInfo' sizes: $ref: '#/components/schemas/MediaSizes' type: @@ -89,8 +87,7 @@ components: MediaExtended: properties: additional_media_info: - additionalProperties: true - type: object + $ref: '#/components/schemas/AdditionalMediaInfo' display_url: format: uri type: string @@ -98,8 +95,7 @@ components: format: uri type: string ext_media_availability: - additionalProperties: true - type: object + $ref: '#/components/schemas/extMediaAvailability' features: type: object id_str: @@ -110,8 +106,7 @@ components: type: integer type: array mediaStats: - additionalProperties: true - type: object + $ref: '#/components/schemas/mediaStats' media_key: type: string media_url_https: @@ -121,9 +116,6 @@ components: $ref: '#/components/schemas/MediaOriginalInfo' sizes: $ref: '#/components/schemas/MediaSizes' - status: - additionalProperties: true - type: object type: type: string url: @@ -451,6 +443,18 @@ components: UserMention: additionalProperties: true type: object + extMediaAvailability: + properties: + status: + type: string + required: + - status + mediaStats: + properties: + viewCount: + type: integer + required: + - viewCount info: title: Twitter OpenAPI version: 0.0.1 diff --git a/dist/compatible/schemas/typename.yaml b/dist/compatible/schemas/typename.yaml index 197eaaa..5262109 100644 --- a/dist/compatible/schemas/typename.yaml +++ b/dist/compatible/schemas/typename.yaml @@ -11,6 +11,7 @@ components: - TweetTombstone - TimelinePrompt - TimelineMessagePrompt + - TweetUnavailable - Tweet - User type: string diff --git a/dist/docs/schemas/tweet.yaml b/dist/docs/schemas/tweet.yaml index c027220..99ac345 100644 --- a/dist/docs/schemas/tweet.yaml +++ b/dist/docs/schemas/tweet.yaml @@ -1,5 +1,11 @@ components: schemas: + AdditionalMediaInfo: + properties: + monetizable: + type: boolean + required: + - monetizable Entities: properties: hashtags: @@ -46,6 +52,8 @@ components: expanded_url: format: uri type: string + features: + type: object id_str: pattern: ^[0-9]+$ type: string @@ -57,17 +65,7 @@ components: format: uri type: string original_info: - properties: - focus_rects: - items: - additionalProperties: true - type: object - type: array - height: - type: integer - width: - type: integer - type: object + $ref: '#/components/schemas/MediaOriginalInfo' sizes: $ref: '#/components/schemas/MediaSizes' type: @@ -89,8 +87,7 @@ components: MediaExtended: properties: additional_media_info: - additionalProperties: true - type: object + $ref: '#/components/schemas/AdditionalMediaInfo' display_url: format: uri type: string @@ -98,8 +95,7 @@ components: format: uri type: string ext_media_availability: - additionalProperties: true - type: object + $ref: '#/components/schemas/extMediaAvailability' features: type: object id_str: @@ -110,8 +106,7 @@ components: type: integer type: array mediaStats: - additionalProperties: true - type: object + $ref: '#/components/schemas/mediaStats' media_key: type: string media_url_https: @@ -121,9 +116,6 @@ components: $ref: '#/components/schemas/MediaOriginalInfo' sizes: $ref: '#/components/schemas/MediaSizes' - status: - additionalProperties: true - type: object type: type: string url: @@ -451,6 +443,18 @@ components: UserMention: additionalProperties: true type: object + extMediaAvailability: + properties: + status: + type: string + required: + - status + mediaStats: + properties: + viewCount: + type: integer + required: + - viewCount info: title: Twitter OpenAPI version: 0.0.1 diff --git a/dist/docs/schemas/typename.yaml b/dist/docs/schemas/typename.yaml index 197eaaa..5262109 100644 --- a/dist/docs/schemas/typename.yaml +++ b/dist/docs/schemas/typename.yaml @@ -11,6 +11,7 @@ components: - TweetTombstone - TimelinePrompt - TimelineMessagePrompt + - TweetUnavailable - Tweet - User type: string From 7414fb6dba0b946677d984ebfdbb549f181e3903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 12:47:26 +0900 Subject: [PATCH 07/24] update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- dist/compatible/schemas/tweet.yaml | 156 ++++++++++++++------------- dist/docs/schemas/tweet.yaml | 156 ++++++++++++++------------- src/openapi/schemas/tweet.yaml | 163 ++++++++++++++++------------- 3 files changed, 256 insertions(+), 219 deletions(-) diff --git a/dist/compatible/schemas/tweet.yaml b/dist/compatible/schemas/tweet.yaml index 99ac345..c18e3fc 100644 --- a/dist/compatible/schemas/tweet.yaml +++ b/dist/compatible/schemas/tweet.yaml @@ -219,6 +219,13 @@ components: required: - content_type - url + SelfThread: + properties: + id_str: + pattern: ^[0-9]+$ + type: string + required: + - id_str Symbol: additionalProperties: true type: object @@ -227,69 +234,13 @@ components: __typename: $ref: ./typename.yaml#/components/schemas/TypeName card: - properties: - legacy: - properties: - binding_values: - items: - properties: - key: - type: string - value: - properties: - boolean_value: - type: boolean - scribe_key: - type: string - string_value: - type: string - type: - type: string - required: - - type - type: object - required: - - key - - value - type: object - type: array - name: - type: string - url: - type: string - required: - - binding_values - - name - - url - type: object - rest_id: - type: string - type: object + $ref: '#/components/schemas/TweetCard' core: $ref: ./user.yaml#/components/schemas/UserResultCore edit_control: - properties: - edit_tweet_ids: - items: - pattern: ^[0-9]+$ - type: string - type: array - editable_until_msecs: - pattern: ^[0-9]+$ - type: string - edits_remaining: - pattern: ^[0-9]+$ - type: string - is_edit_eligible: - type: boolean - type: object + $ref: '#/components/schemas/TweetEditControl' edit_prespective: - properties: - favorited: - type: boolean - retweeted: - type: boolean - type: object + $ref: '#/components/schemas/TweetEditPrespective' is_translatable: default: false type: boolean @@ -304,13 +255,7 @@ components: additionalProperties: true type: object views: - properties: - count: - pattern: ^[0-9]+$ - type: string - state: - type: string - type: object + $ref: '#/components/schemas/TweetView' required: - rest_id - core @@ -318,6 +263,68 @@ components: - is_translatable - legacy - views + TweetCard: + properties: + legacy: + $ref: '#/components/schemas/TweetCardLegacy' + rest_id: + type: string + TweetCardLegacy: + properties: + binding_values: + items: + $ref: '#/components/schemas/TweetCardLegacyBindingValue' + type: array + name: + type: string + url: + type: string + required: + - binding_values + - name + - url + TweetCardLegacyBindingValue: + properties: + key: + type: string + value: + $ref: '#/components/schemas/TweetCardLegacyBindingValueData' + required: + - key + - value + TweetCardLegacyBindingValueData: + properties: + boolean_value: + type: boolean + scribe_key: + type: string + string_value: + type: string + type: + type: string + required: + - type + TweetEditControl: + properties: + edit_tweet_ids: + items: + pattern: ^[0-9]+$ + type: string + type: array + editable_until_msecs: + pattern: ^[0-9]+$ + type: string + edits_remaining: + pattern: ^[0-9]+$ + type: string + is_edit_eligible: + type: boolean + TweetEditPrespective: + properties: + favorited: + type: boolean + retweeted: + type: boolean TweetLegacy: properties: bookmark_count: @@ -367,13 +374,7 @@ components: retweeted_status_result: $ref: ./content.yaml#/components/schemas/ItemResult self_thread: - properties: - id_str: - pattern: ^[0-9]+$ - type: string - required: - - id_str - type: object + $ref: '#/components/schemas/SelfThread' user_id_str: pattern: ^[0-9]+$ type: string @@ -411,6 +412,15 @@ components: - $ref: '#/components/schemas/Tweet' - $ref: '#/components/schemas/TweetWithVisibilityResults' - $ref: '#/components/schemas/TweetTombstone' + TweetView: + properties: + count: + pattern: ^[0-9]+$ + type: string + state: + type: string + required: + - state TweetWithVisibilityResults: properties: __typename: diff --git a/dist/docs/schemas/tweet.yaml b/dist/docs/schemas/tweet.yaml index 99ac345..c18e3fc 100644 --- a/dist/docs/schemas/tweet.yaml +++ b/dist/docs/schemas/tweet.yaml @@ -219,6 +219,13 @@ components: required: - content_type - url + SelfThread: + properties: + id_str: + pattern: ^[0-9]+$ + type: string + required: + - id_str Symbol: additionalProperties: true type: object @@ -227,69 +234,13 @@ components: __typename: $ref: ./typename.yaml#/components/schemas/TypeName card: - properties: - legacy: - properties: - binding_values: - items: - properties: - key: - type: string - value: - properties: - boolean_value: - type: boolean - scribe_key: - type: string - string_value: - type: string - type: - type: string - required: - - type - type: object - required: - - key - - value - type: object - type: array - name: - type: string - url: - type: string - required: - - binding_values - - name - - url - type: object - rest_id: - type: string - type: object + $ref: '#/components/schemas/TweetCard' core: $ref: ./user.yaml#/components/schemas/UserResultCore edit_control: - properties: - edit_tweet_ids: - items: - pattern: ^[0-9]+$ - type: string - type: array - editable_until_msecs: - pattern: ^[0-9]+$ - type: string - edits_remaining: - pattern: ^[0-9]+$ - type: string - is_edit_eligible: - type: boolean - type: object + $ref: '#/components/schemas/TweetEditControl' edit_prespective: - properties: - favorited: - type: boolean - retweeted: - type: boolean - type: object + $ref: '#/components/schemas/TweetEditPrespective' is_translatable: default: false type: boolean @@ -304,13 +255,7 @@ components: additionalProperties: true type: object views: - properties: - count: - pattern: ^[0-9]+$ - type: string - state: - type: string - type: object + $ref: '#/components/schemas/TweetView' required: - rest_id - core @@ -318,6 +263,68 @@ components: - is_translatable - legacy - views + TweetCard: + properties: + legacy: + $ref: '#/components/schemas/TweetCardLegacy' + rest_id: + type: string + TweetCardLegacy: + properties: + binding_values: + items: + $ref: '#/components/schemas/TweetCardLegacyBindingValue' + type: array + name: + type: string + url: + type: string + required: + - binding_values + - name + - url + TweetCardLegacyBindingValue: + properties: + key: + type: string + value: + $ref: '#/components/schemas/TweetCardLegacyBindingValueData' + required: + - key + - value + TweetCardLegacyBindingValueData: + properties: + boolean_value: + type: boolean + scribe_key: + type: string + string_value: + type: string + type: + type: string + required: + - type + TweetEditControl: + properties: + edit_tweet_ids: + items: + pattern: ^[0-9]+$ + type: string + type: array + editable_until_msecs: + pattern: ^[0-9]+$ + type: string + edits_remaining: + pattern: ^[0-9]+$ + type: string + is_edit_eligible: + type: boolean + TweetEditPrespective: + properties: + favorited: + type: boolean + retweeted: + type: boolean TweetLegacy: properties: bookmark_count: @@ -367,13 +374,7 @@ components: retweeted_status_result: $ref: ./content.yaml#/components/schemas/ItemResult self_thread: - properties: - id_str: - pattern: ^[0-9]+$ - type: string - required: - - id_str - type: object + $ref: '#/components/schemas/SelfThread' user_id_str: pattern: ^[0-9]+$ type: string @@ -411,6 +412,15 @@ components: - $ref: '#/components/schemas/Tweet' - $ref: '#/components/schemas/TweetWithVisibilityResults' - $ref: '#/components/schemas/TweetTombstone' + TweetView: + properties: + count: + pattern: ^[0-9]+$ + type: string + state: + type: string + required: + - state TweetWithVisibilityResults: properties: __typename: diff --git a/src/openapi/schemas/tweet.yaml b/src/openapi/schemas/tweet.yaml index 73dd612..ba30666 100644 --- a/src/openapi/schemas/tweet.yaml +++ b/src/openapi/schemas/tweet.yaml @@ -54,86 +54,101 @@ components: core: $ref: "./user.yaml#/components/schemas/UserResultCore" card: - type: object - properties: - rest_id: - type: string - legacy: - type: object - required: - - "binding_values" - - "name" - - "url" - properties: - name: - type: string - url: - type: string - binding_values: - type: array - items: - type: object - required: - - "key" - - "value" - properties: - key: - type: string - value: - type: object - required: - - "type" - properties: - string_value: - type: string - boolean_value: - type: boolean - scribe_key: - type: string - type: - type: string + $ref: "#/components/schemas/TweetCard" unmention_data: type: object additionalProperties: true # todo edit_control: - type: object - properties: - edit_tweet_ids: - type: array - items: - type: string - pattern: "^[0-9]+$" - editable_until_msecs: - type: string - pattern: "^[0-9]+$" - is_edit_eligible: - type: boolean - edits_remaining: - type: string - pattern: "^[0-9]+$" + $ref: "#/components/schemas/TweetEditControl" edit_prespective: - type: object - properties: - favorited: - type: boolean - retweeted: - type: boolean + $ref: "#/components/schemas/TweetEditPrespective" is_translatable: type: boolean default: false legacy: $ref: "#/components/schemas/TweetLegacy" views: - type: object - properties: - count: - type: string - pattern: "^[0-9]+$" - state: - type: string # enum + $ref: "#/components/schemas/TweetView" quoted_status_result: $ref: "./content.yaml#/components/schemas/ItemResult" + TweetEditControl: + properties: + edit_tweet_ids: + type: array + items: + type: string + pattern: "^[0-9]+$" + editable_until_msecs: + type: string + pattern: "^[0-9]+$" + is_edit_eligible: + type: boolean + edits_remaining: + type: string + pattern: "^[0-9]+$" + + TweetEditPrespective: + properties: + favorited: + type: boolean + retweeted: + type: boolean + TweetView: + required: + - "state" + properties: + count: + type: string + pattern: "^[0-9]+$" + state: + type: string # enum + + TweetCard: + properties: + rest_id: + type: string + legacy: + $ref: "#/components/schemas/TweetCardLegacy" + + TweetCardLegacy: + required: + - "binding_values" + - "name" + - "url" + properties: + name: + type: string + url: + type: string + binding_values: + type: array + items: + $ref: "#/components/schemas/TweetCardLegacyBindingValue" + + TweetCardLegacyBindingValue: + required: + - "key" + - "value" + properties: + key: + type: string + value: + $ref: "#/components/schemas/TweetCardLegacyBindingValueData" + + TweetCardLegacyBindingValueData: + required: + - "type" + properties: + string_value: + type: string + boolean_value: + type: boolean + scribe_key: + type: string + type: + type: string + TweetLegacy: required: - "bookmark_count" @@ -202,16 +217,18 @@ components: type: string pattern: "^[0-9]+$" self_thread: - type: object - required: - - "id_str" - properties: - id_str: - type: string - pattern: "^[0-9]+$" + $ref: "#/components/schemas/SelfThread" extended_entities: $ref: "#/components/schemas/ExtendedEntities" + SelfThread: + required: + - "id_str" + properties: + id_str: + type: string + pattern: "^[0-9]+$" + Entities: required: - "hashtags" From d37db1287fbc813489aa065ec66f58d1a3849017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 13:40:34 +0900 Subject: [PATCH 08/24] add cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- .gitignore | 3 ++- test/python/test_serialize.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 5e63e49..a7e4882 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ __pycache__/ cookie.json *_generated -*.jar \ No newline at end of file +*.jar +cache/ \ No newline at end of file diff --git a/test/python/test_serialize.py b/test/python/test_serialize.py index a746ace..d490753 100644 --- a/test/python/test_serialize.py +++ b/test/python/test_serialize.py @@ -3,10 +3,9 @@ import os import logging import base64 import openapi_client as pt -from pydantic import BaseModel from pathlib import Path import time - +import glob logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s") logger = logging.getLogger("test_serialize") @@ -75,6 +74,21 @@ def get_kwargs(key, additional): return kwargs +def save_cache(data): + rand = time.time_ns() + os.makedirs("cache", exist_ok=True) + with open(f"cache/{rand}.json", "w+") as f: + json.dump(data, f, indent=4) + + +for file in glob.glob("cache/*.json"): + with open(file, "r") as f: + cache = json.load(f) + data = pt.__dict__[cache["type"]].from_json(cache["raw"]) + +logger.info(f"Success: {len(glob.glob('cache/*.json'))}") + + api_conf = pt.Configuration( api_key={ "ClientLanguage": "en", @@ -95,10 +109,10 @@ for x in [pt.DefaultApi, pt.TweetApi, pt.UserApi, pt.UserListApi]: for props, fn in x.__dict__.items(): if not callable(fn): continue - if props.startswith("__") or props.endswith("_with_http_info"): + if props.startswith("__") or not props.endswith("_with_http_info"): continue - key = get_key(props) + key = get_key(props[:-17]) cursor_list = set([None]) cursor_history = set() @@ -109,14 +123,17 @@ for x in [pt.DefaultApi, pt.TweetApi, pt.UserApi, pt.UserListApi]: logger.info(f"Try: {key} {cursor}") kwargs = get_kwargs(key, {} if cursor is None else {"cursor": cursor}) - res: dict = getattr(x(api_client), props)(**kwargs).to_dict() + res: pt.ApiResponse = getattr(x(api_client), props)(**kwargs) + data = res.data.to_dict() - new_cursor = set(get_cursor(res, find_cursor)) - cursor_history + save_cache({"raw": res.raw_data, "type": res.data.__class__.__name__}) + + new_cursor = set(get_cursor(data, find_cursor)) - cursor_history cursor_list.update(new_cursor) # logger.info(f"Find cursor: {len(new_cursor)}") - if res.get("errors") is not None: - logger.error(res) + if data.get("errors") is not None: + logger.error(data) error_count += 1 if len(cursor_list) == 0: From 3919f8f5c8766bad75e87e86adaea0dcb7d585c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 13:58:06 +0900 Subject: [PATCH 09/24] fix test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- test/python/test_serialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/python/test_serialize.py b/test/python/test_serialize.py index d490753..4a1435e 100644 --- a/test/python/test_serialize.py +++ b/test/python/test_serialize.py @@ -112,7 +112,7 @@ for x in [pt.DefaultApi, pt.TweetApi, pt.UserApi, pt.UserListApi]: if props.startswith("__") or not props.endswith("_with_http_info"): continue - key = get_key(props[:-17]) + key = get_key(props[:-15]) cursor_list = set([None]) cursor_history = set() From 8a7eba499fdc9db25fed0793bc1bafacbf63e34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 13:58:16 +0900 Subject: [PATCH 10/24] enum value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/paths/other.yaml | 3 ++- src/openapi/schemas/content.yaml | 6 ++++-- src/openapi/schemas/tweet.yaml | 16 ++++++++++------ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/openapi/paths/other.yaml b/src/openapi/paths/other.yaml index 280b3a8..de47a86 100644 --- a/src/openapi/paths/other.yaml +++ b/src/openapi/paths/other.yaml @@ -69,7 +69,8 @@ components: superFollowersCount: type: integer superFollowsApplicationStatus: - type: string # enum: NotStarted + type: string + enum: [NotStarted] user_id: type: string pattern: "^[0-9]+$" diff --git a/src/openapi/schemas/content.yaml b/src/openapi/schemas/content.yaml index 1d91d6d..913339f 100644 --- a/src/openapi/schemas/content.yaml +++ b/src/openapi/schemas/content.yaml @@ -60,7 +60,8 @@ components: entryType: $ref: "#/components/schemas/ContentEntryType" # TimelineTimelineCursor displayType: - type: string # enum + type: string + enum: [Vertical, VerticalConversation] items: type: array items: @@ -183,7 +184,8 @@ components: SocialContext: $ref: "#/components/schemas/SocialContext" userDisplayType: - type: string # enum + type: string + enum: [User, UserDetailed] user_results: $ref: "./user.yaml#/components/schemas/UserResults" diff --git a/src/openapi/schemas/tweet.yaml b/src/openapi/schemas/tweet.yaml index ba30666..e6ef176 100644 --- a/src/openapi/schemas/tweet.yaml +++ b/src/openapi/schemas/tweet.yaml @@ -102,7 +102,8 @@ components: type: string pattern: "^[0-9]+$" state: - type: string # enum + type: string + enum: [EnabledWithCount] TweetCard: properties: @@ -317,7 +318,8 @@ components: type: string format: uri type: - type: string # enum photo + type: string + enum: [photo] url: type: string format: uri @@ -371,7 +373,8 @@ components: type: string format: uri type: - type: string # enum # photo | video + type: string + enum: [photo, video, animated_gif] url: type: string format: uri @@ -469,7 +472,8 @@ components: - "status" properties: status: - type: string # enum Available + type: string + enum: [Available] MediaSizes: required: @@ -498,5 +502,5 @@ components: h: type: integer resize: - type: string # enum fit - + type: string + enum: [crop, fit] From 214392ec9ba0cca9bf2996119e499a22ceb2c764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 13:58:21 +0900 Subject: [PATCH 11/24] build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- dist/compatible/paths/other.yaml | 2 ++ dist/compatible/schemas/content.yaml | 6 ++++++ dist/compatible/schemas/tweet.yaml | 13 +++++++++++++ dist/docs/paths/other.yaml | 2 ++ dist/docs/schemas/content.yaml | 6 ++++++ dist/docs/schemas/tweet.yaml | 13 +++++++++++++ 6 files changed, 42 insertions(+) diff --git a/dist/compatible/paths/other.yaml b/dist/compatible/paths/other.yaml index 0d28120..02729a8 100644 --- a/dist/compatible/paths/other.yaml +++ b/dist/compatible/paths/other.yaml @@ -47,6 +47,8 @@ components: superFollowersCount: type: integer superFollowsApplicationStatus: + enum: + - NotStarted type: string userFeatures: $ref: '#/components/schemas/UserFeatures' diff --git a/dist/compatible/schemas/content.yaml b/dist/compatible/schemas/content.yaml index cf53767..da479f4 100644 --- a/dist/compatible/schemas/content.yaml +++ b/dist/compatible/schemas/content.yaml @@ -138,6 +138,9 @@ components: additionalProperties: true type: object displayType: + enum: + - Vertical + - VerticalConversation type: string entryType: $ref: '#/components/schemas/ContentEntryType' @@ -185,6 +188,9 @@ components: itemType: $ref: '#/components/schemas/ContentItemType' userDisplayType: + enum: + - User + - UserDetailed type: string user_results: $ref: ./user.yaml#/components/schemas/UserResults diff --git a/dist/compatible/schemas/tweet.yaml b/dist/compatible/schemas/tweet.yaml index c18e3fc..5fe8e20 100644 --- a/dist/compatible/schemas/tweet.yaml +++ b/dist/compatible/schemas/tweet.yaml @@ -69,6 +69,8 @@ components: sizes: $ref: '#/components/schemas/MediaSizes' type: + enum: + - photo type: string url: format: uri @@ -117,6 +119,10 @@ components: sizes: $ref: '#/components/schemas/MediaSizes' type: + enum: + - photo + - video + - animated_gif type: string url: format: uri @@ -170,6 +176,9 @@ components: h: type: integer resize: + enum: + - crop + - fit type: string w: type: integer @@ -418,6 +427,8 @@ components: pattern: ^[0-9]+$ type: string state: + enum: + - EnabledWithCount type: string required: - state @@ -456,6 +467,8 @@ components: extMediaAvailability: properties: status: + enum: + - Available type: string required: - status diff --git a/dist/docs/paths/other.yaml b/dist/docs/paths/other.yaml index 0d28120..02729a8 100644 --- a/dist/docs/paths/other.yaml +++ b/dist/docs/paths/other.yaml @@ -47,6 +47,8 @@ components: superFollowersCount: type: integer superFollowsApplicationStatus: + enum: + - NotStarted type: string userFeatures: $ref: '#/components/schemas/UserFeatures' diff --git a/dist/docs/schemas/content.yaml b/dist/docs/schemas/content.yaml index cf53767..da479f4 100644 --- a/dist/docs/schemas/content.yaml +++ b/dist/docs/schemas/content.yaml @@ -138,6 +138,9 @@ components: additionalProperties: true type: object displayType: + enum: + - Vertical + - VerticalConversation type: string entryType: $ref: '#/components/schemas/ContentEntryType' @@ -185,6 +188,9 @@ components: itemType: $ref: '#/components/schemas/ContentItemType' userDisplayType: + enum: + - User + - UserDetailed type: string user_results: $ref: ./user.yaml#/components/schemas/UserResults diff --git a/dist/docs/schemas/tweet.yaml b/dist/docs/schemas/tweet.yaml index c18e3fc..5fe8e20 100644 --- a/dist/docs/schemas/tweet.yaml +++ b/dist/docs/schemas/tweet.yaml @@ -69,6 +69,8 @@ components: sizes: $ref: '#/components/schemas/MediaSizes' type: + enum: + - photo type: string url: format: uri @@ -117,6 +119,10 @@ components: sizes: $ref: '#/components/schemas/MediaSizes' type: + enum: + - photo + - video + - animated_gif type: string url: format: uri @@ -170,6 +176,9 @@ components: h: type: integer resize: + enum: + - crop + - fit type: string w: type: integer @@ -418,6 +427,8 @@ components: pattern: ^[0-9]+$ type: string state: + enum: + - EnabledWithCount type: string required: - state @@ -456,6 +467,8 @@ components: extMediaAvailability: properties: status: + enum: + - Available type: string required: - status From bc05f30d3a1f7eb02f69d6698db2fb5fe5976fb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 15:17:17 +0900 Subject: [PATCH 12/24] update test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- test/python/test_serialize.py | 48 +++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/test/python/test_serialize.py b/test/python/test_serialize.py index 4a1435e..8839a56 100644 --- a/test/python/test_serialize.py +++ b/test/python/test_serialize.py @@ -6,6 +6,7 @@ import openapi_client as pt from pathlib import Path import time import glob +import aenum logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s") logger = logging.getLogger("test_serialize") @@ -14,7 +15,7 @@ TWITTER_SESSION = os.environ.get("TWITTER_SESSION", None) ERROR_UNCATCHED = os.environ.get("ERROR_UNCATCHED", "false").lower() == "true" SLEEP_TIME = float(os.environ.get("SLEEP", "0")) CUESOR_TEST_COUNT = int(os.environ.get("CUESOR_TEST_COUNT", "3")) - +STRICT_MODE = os.environ.get("STRICT_MODE", "false").lower() == "true" if Path("cookie.json").exists(): with open("cookie.json", "r") as f: @@ -74,6 +75,44 @@ def get_kwargs(key, additional): return kwargs +def match_rate_zero(key): + if STRICT_MODE: + raise Exception(f"Strict mode: {key}") + return 0 + + +def match_rate(a, b, key=""): + if isinstance(a, aenum.Enum): + a = a.value + if isinstance(b, aenum.Enum): + b = b.value + if a is None and b is False: + return 1 + if a is False and b is None: + return 1 + if type(a) != type(b): + return match_rate_zero(key) + if type(a) == dict and type(b) == dict: + if len(a) == 0 and len(b) == 0: + return 1 + if len(a) == 0 or len(b) == 0: + return match_rate_zero(key) + + data = [match_rate(a.get(k), b.get(k), key=f"{key}.{k}") for k in a.keys()] + + return sum(data) / len(a) + if type(a) == list and type(b) == list: + if len(a) == 0 and len(b) == 0: + return 1 + if len(a) != len(b): + return match_rate_zero(a, b, key=key) + data = [match_rate(a[i], b[i], key=f"{key}[{i}]") for i in range(len(a))] + return sum(data) / len(a) + if a == b: + return 1 + return match_rate_zero(a, b, key=key) + + def save_cache(data): rand = time.time_ns() os.makedirs("cache", exist_ok=True) @@ -81,10 +120,13 @@ def save_cache(data): json.dump(data, f, indent=4) +# ====== Load cache ====== for file in glob.glob("cache/*.json"): with open(file, "r") as f: cache = json.load(f) data = pt.__dict__[cache["type"]].from_json(cache["raw"]) + rate = match_rate(data.to_dict(), json.loads(cache["raw"])) + logger.info(f"Match rate: {rate}") logger.info(f"Success: {len(glob.glob('cache/*.json'))}") @@ -130,7 +172,9 @@ for x in [pt.DefaultApi, pt.TweetApi, pt.UserApi, pt.UserListApi]: new_cursor = set(get_cursor(data, find_cursor)) - cursor_history cursor_list.update(new_cursor) - # logger.info(f"Find cursor: {len(new_cursor)}") + + rate = match_rate(data, json.loads(res.raw_data)) + logger.info(f"Match rate: {rate}") if data.get("errors") is not None: logger.error(data) From 7f3270cf8abad4b0ea4a82aad180130d3db64d37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 15:17:40 +0900 Subject: [PATCH 13/24] add TweetEditControl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/content.yaml | 2 +- src/openapi/schemas/tweet.yaml | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/openapi/schemas/content.yaml b/src/openapi/schemas/content.yaml index 913339f..95508a9 100644 --- a/src/openapi/schemas/content.yaml +++ b/src/openapi/schemas/content.yaml @@ -185,7 +185,7 @@ components: $ref: "#/components/schemas/SocialContext" userDisplayType: type: string - enum: [User, UserDetailed] + enum: [User, UserDetailed, SubscribableUser] user_results: $ref: "./user.yaml#/components/schemas/UserResults" diff --git a/src/openapi/schemas/tweet.yaml b/src/openapi/schemas/tweet.yaml index e6ef176..3a1254d 100644 --- a/src/openapi/schemas/tweet.yaml +++ b/src/openapi/schemas/tweet.yaml @@ -40,8 +40,8 @@ components: - "rest_id" - "core" - "edit_control" - # - "edit_prespective" - "is_translatable" + - "source" - "legacy" - "views" @@ -65,6 +65,8 @@ components: is_translatable: type: boolean default: false + source: + type: string # html (Twitter for Android) legacy: $ref: "#/components/schemas/TweetLegacy" views: @@ -87,6 +89,32 @@ components: edits_remaining: type: string pattern: "^[0-9]+$" + initial_tweet_id: + type: string + pattern: "^[0-9]+$" + edit_control_initial: + $ref: "#/components/schemas/TweetEditControlInitial" + + TweetEditControlInitial: + required: + - "edit_tweet_ids" + - "editable_until_msecs" + - "is_edit_eligible" + - "edits_remaining" + properties: + edit_tweet_ids: + type: array + items: + type: string + pattern: "^[0-9]+$" + editable_until_msecs: + type: string + pattern: "^[0-9]+$" + is_edit_eligible: + type: boolean + edits_remaining: + type: string + pattern: "^[0-9]+$" TweetEditPrespective: properties: From 8917dcf30dbf1399d1e3a3de12d7d8d7764a7932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 15:17:46 +0900 Subject: [PATCH 14/24] build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- dist/compatible/schemas/content.yaml | 1 + dist/compatible/schemas/tweet.yaml | 28 ++++++++++++++++++++++++++++ dist/docs/schemas/content.yaml | 1 + dist/docs/schemas/tweet.yaml | 28 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+) diff --git a/dist/compatible/schemas/content.yaml b/dist/compatible/schemas/content.yaml index da479f4..ccadf95 100644 --- a/dist/compatible/schemas/content.yaml +++ b/dist/compatible/schemas/content.yaml @@ -191,6 +191,7 @@ components: enum: - User - UserDetailed + - SubscribableUser type: string user_results: $ref: ./user.yaml#/components/schemas/UserResults diff --git a/dist/compatible/schemas/tweet.yaml b/dist/compatible/schemas/tweet.yaml index 5fe8e20..c115145 100644 --- a/dist/compatible/schemas/tweet.yaml +++ b/dist/compatible/schemas/tweet.yaml @@ -260,6 +260,8 @@ components: rest_id: pattern: ^[0-9]+$ type: string + source: + type: string unmention_data: additionalProperties: true type: object @@ -270,6 +272,7 @@ components: - core - edit_control - is_translatable + - source - legacy - views TweetCard: @@ -314,6 +317,26 @@ components: required: - type TweetEditControl: + properties: + edit_control_initial: + $ref: '#/components/schemas/TweetEditControlInitial' + edit_tweet_ids: + items: + pattern: ^[0-9]+$ + type: string + type: array + editable_until_msecs: + pattern: ^[0-9]+$ + type: string + edits_remaining: + pattern: ^[0-9]+$ + type: string + initial_tweet_id: + pattern: ^[0-9]+$ + type: string + is_edit_eligible: + type: boolean + TweetEditControlInitial: properties: edit_tweet_ids: items: @@ -328,6 +351,11 @@ components: type: string is_edit_eligible: type: boolean + required: + - edit_tweet_ids + - editable_until_msecs + - is_edit_eligible + - edits_remaining TweetEditPrespective: properties: favorited: diff --git a/dist/docs/schemas/content.yaml b/dist/docs/schemas/content.yaml index da479f4..ccadf95 100644 --- a/dist/docs/schemas/content.yaml +++ b/dist/docs/schemas/content.yaml @@ -191,6 +191,7 @@ components: enum: - User - UserDetailed + - SubscribableUser type: string user_results: $ref: ./user.yaml#/components/schemas/UserResults diff --git a/dist/docs/schemas/tweet.yaml b/dist/docs/schemas/tweet.yaml index 5fe8e20..c115145 100644 --- a/dist/docs/schemas/tweet.yaml +++ b/dist/docs/schemas/tweet.yaml @@ -260,6 +260,8 @@ components: rest_id: pattern: ^[0-9]+$ type: string + source: + type: string unmention_data: additionalProperties: true type: object @@ -270,6 +272,7 @@ components: - core - edit_control - is_translatable + - source - legacy - views TweetCard: @@ -314,6 +317,26 @@ components: required: - type TweetEditControl: + properties: + edit_control_initial: + $ref: '#/components/schemas/TweetEditControlInitial' + edit_tweet_ids: + items: + pattern: ^[0-9]+$ + type: string + type: array + editable_until_msecs: + pattern: ^[0-9]+$ + type: string + edits_remaining: + pattern: ^[0-9]+$ + type: string + initial_tweet_id: + pattern: ^[0-9]+$ + type: string + is_edit_eligible: + type: boolean + TweetEditControlInitial: properties: edit_tweet_ids: items: @@ -328,6 +351,11 @@ components: type: string is_edit_eligible: type: boolean + required: + - edit_tweet_ids + - editable_until_msecs + - is_edit_eligible + - edits_remaining TweetEditPrespective: properties: favorited: From bc54262db6d917e9c8b47ee95dbee5c406ce195a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 1 Sep 2023 15:37:17 +0900 Subject: [PATCH 15/24] add rate log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- test/python/test_serialize.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/python/test_serialize.py b/test/python/test_serialize.py index 2eb9982..82de613 100644 --- a/test/python/test_serialize.py +++ b/test/python/test_serialize.py @@ -203,8 +203,12 @@ for x in [pt.DefaultApi, pt.TweetApi, pt.UserApi, pt.UsersApi, pt.UserListApi]: try: logger.info(f"Try: Self UserByScreenName Test") kwargs = get_kwargs("UserByScreenName", {"screen_name": "a810810931931"}) - res = pt.UserApi(api_client).get_user_by_screen_name(**kwargs).to_dict() - if not res["data"]["user"]["result"]["legacy"]["screen_name"] == "a810810931931": + res = pt.UserApi(api_client).get_user_by_screen_name_with_http_info(**kwargs) + data = res.data.to_dict() + + rate = match_rate(data, json.loads(res.raw_data)) + logger.info(f"Match rate: {rate}") + if not data["data"]["user"]["result"]["legacy"]["screen_name"] == "a810810931931": raise Exception("UserByScreenName failed") except Exception as e: error_dump(e) @@ -213,7 +217,12 @@ except Exception as e: try: logger.info(f"Try: Self UserTweets Test") kwargs = get_kwargs("UserTweets", {"userId": "1180389371481976833"}) - pt.TweetApi(api_client).get_user_tweets(**kwargs) + res = pt.TweetApi(api_client).get_user_tweets_with_http_info(**kwargs) + data = res.data.to_dict() + + rate = match_rate(data, json.loads(res.raw_data)) + logger.info(f"Match rate: {rate}") + except Exception as e: error_dump(e) error_count += 1 From f8e41919366b5bbe02d5b93f3edd7f02bf2cf114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Tue, 12 Sep 2023 04:32:08 +0900 Subject: [PATCH 16/24] wip add TimelineShowCover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/content.yaml | 11 +++ src/openapi/schemas/instruction.yaml | 133 +++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) diff --git a/src/openapi/schemas/content.yaml b/src/openapi/schemas/content.yaml index 95508a9..fdd72fd 100644 --- a/src/openapi/schemas/content.yaml +++ b/src/openapi/schemas/content.yaml @@ -219,3 +219,14 @@ components: __typename: $ref: "./typename.yaml#/components/schemas/TypeName" # TimelineMessagePrompt additionalProperties: true # todo + + ClientEventInfo: + required: + - component + - elemen + properties: + component: + type: string # enum half_cover + element: + type: string + pattern: "^(january|february|march|april|may|june|july|august|september|october|november|december)-[0-9]{4}-([a-z]-)+[a-z]+$" # august-2023-privacy-prompt-candidate diff --git a/src/openapi/schemas/instruction.yaml b/src/openapi/schemas/instruction.yaml index 881f9dd..73bb9e6 100644 --- a/src/openapi/schemas/instruction.yaml +++ b/src/openapi/schemas/instruction.yaml @@ -15,6 +15,8 @@ components: - $ref: "#/components/schemas/TimelineReplaceEntry" - $ref: "#/components/schemas/TimelineShowAlert" - $ref: "#/components/schemas/TimelineTerminateTimeline" + - $ref: "#/components/schemas/TimelineShowCover" + discriminator: propertyName: type mapping": # deprecated @@ -25,6 +27,7 @@ components: TimelineReplaceEntry: "#/components/schemas/TimelineReplaceEntry" TimelineShowAlert: "#/components/schemas/TimelineShowAlert" TimelineTerminateTimeline: "#/components/schemas/TimelineTerminateTimeline" + TimelineShowCover: "#/components/schemas/TimelineShowCover" InstructionType: type: string @@ -37,6 +40,7 @@ components: TimelineReplaceEntry, TimelineShowAlert, TimelineTerminateTimeline, + TimelineShowCover, ] TimelineAddEntries: @@ -162,3 +166,132 @@ components: sortIndex: type: string pattern: "[0-9]+$" + + TimelineShowCover: + required: + - type + - clientEventInfo + - cover + properties: + type: + $ref: "#/components/schemas/InstructionType" # TimelineShowCover + clientEventInfo: + $ref: "./content.yaml#/components/schemas/ClientEventInfo" + cover: + $ref: "#/components/schemas/TimelineHalfCover" + + TimelineHalfCover: + required: + - type + - halfCoverDisplayType + - primaryText + - primaryCoverCta + - secondaryText + - impressionCallbacks + - dismissible + properties: + type: + type: string + enum: [TimelineHalfCover] + halfCoverDisplayType: + type: string + enum: [Cover] + primaryText: + $ref: "#/components/schemas/Text" + secondaryText: + $ref: "#/components/schemas/Text" + primaryCoverCta: + $ref: "#/components/schemas/CoverCta" + impressionCallbacks: + type: array + items: + $ref: "#/components/schemas/Callback" + dismissible: + type: boolean + + Text: + required: + - text + - entities + properties: + text: + type: string + entities: + type: array + items: + $ref: "#/components/schemas/TextEntity" + + TextEntity: + required: + - fromIndex + - toIndex + - ref + properties: + fromIndex: + type: integer + toIndex: + type: integer + ref: + $ref: "#/components/schemas/TextEntityRef" + + TextEntityRef: + required: + - type + - url + - urlType + properties: + type: + type: string + enum: [TimelineUrl] + url: + type: string + format: uri + urlType: + type: string + enum: [ExternalUrl] + + CoverCta: + required: + - text + - ctaBehavior + - callbacks + - clientEventInfo + - buttonStyle + properties: + Text: + type: string + ctaBehavior: + $ref: "#/components/schemas/TimelineCoverBehavior" + callbacks: + type: array + items: + $ref: "#/components/schemas/Callback" + clientEventInfo: + $ref: "#/components/schemas/ClientEventInfo" + buttonStyle: + type: string + enum: ["Primary"] + + TimelineCoverBehavior: + required: + - type + properties: + type: + type: string + enum: [TimelineCoverBehaviorDismiss] + + Callback: + required: + - endpoint + properties: + endpoint: + type: string + # pattern: '^/1\.1/[a-z]+/[a-z]+\.json\?[a-z_]+=[a-z0-9-]+(&[a-z_]+=[a-z0-9-]+)+?$' #/1.1/onboarding/fatigue.json?{params} + + ClientEventInfo: + required: + - action + properties: + action: + type: string + enum: [primary_cta] From 14a5290ebaaa80b68e5157ba7e1f191d32b3a841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Tue, 12 Sep 2023 11:52:00 +0900 Subject: [PATCH 17/24] fix Media MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/tweet.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/openapi/schemas/tweet.yaml b/src/openapi/schemas/tweet.yaml index 3a1254d..1fafda0 100644 --- a/src/openapi/schemas/tweet.yaml +++ b/src/openapi/schemas/tweet.yaml @@ -325,7 +325,7 @@ components: - "media_url_https" - "type" - "url" - - "features" + # - "features" - "sizes" - "original_info" properties: @@ -347,7 +347,7 @@ components: format: uri type: type: string - enum: [photo] + enum: [photo, video, animated_gif] url: type: string format: uri @@ -378,7 +378,7 @@ components: - "type" - "url" - "ext_media_availability" - - "features" + # - "features" - "sizes" - "original_info" properties: From 59ec5b51a9dd9789a978a26cd4f6c4430c04efd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Tue, 12 Sep 2023 11:52:18 +0900 Subject: [PATCH 18/24] build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- dist/compatible/schemas/tweet.yaml | 4 ++-- dist/docs/schemas/tweet.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dist/compatible/schemas/tweet.yaml b/dist/compatible/schemas/tweet.yaml index c115145..454ee82 100644 --- a/dist/compatible/schemas/tweet.yaml +++ b/dist/compatible/schemas/tweet.yaml @@ -71,6 +71,8 @@ components: type: enum: - photo + - video + - animated_gif type: string url: format: uri @@ -83,7 +85,6 @@ components: - media_url_https - type - url - - features - sizes - original_info MediaExtended: @@ -139,7 +140,6 @@ components: - type - url - ext_media_availability - - features - sizes - original_info MediaOriginalInfo: diff --git a/dist/docs/schemas/tweet.yaml b/dist/docs/schemas/tweet.yaml index c115145..454ee82 100644 --- a/dist/docs/schemas/tweet.yaml +++ b/dist/docs/schemas/tweet.yaml @@ -71,6 +71,8 @@ components: type: enum: - photo + - video + - animated_gif type: string url: format: uri @@ -83,7 +85,6 @@ components: - media_url_https - type - url - - features - sizes - original_info MediaExtended: @@ -139,7 +140,6 @@ components: - type - url - ext_media_availability - - features - sizes - original_info MediaOriginalInfo: From b7fcdcc6d184b2bf55bc644713a79839c0f7cf99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Tue, 12 Sep 2023 15:16:14 +0900 Subject: [PATCH 19/24] add multi thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- test/python/test_serialize.py | 202 ++++++++++++++++++---------------- 1 file changed, 110 insertions(+), 92 deletions(-) diff --git a/test/python/test_serialize.py b/test/python/test_serialize.py index 82de613..bd7f518 100644 --- a/test/python/test_serialize.py +++ b/test/python/test_serialize.py @@ -7,6 +7,8 @@ from pathlib import Path import time import glob import aenum +import concurrent.futures + logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s") logger = logging.getLogger("test_serialize") @@ -17,22 +19,6 @@ SLEEP_TIME = float(os.environ.get("SLEEP", "0")) CUESOR_TEST_COUNT = int(os.environ.get("CUESOR_TEST_COUNT", "3")) STRICT_MODE = os.environ.get("STRICT_MODE", "false").lower() == "true" -if Path("cookie.json").exists(): - with open("cookie.json", "r") as f: - cookies = json.load(f) -elif TWITTER_SESSION is not None: - data = base64.b64decode(TWITTER_SESSION).decode("utf-8") - cookies = json.loads(data) -else: - commands = ["python -m pip install tweepy_authlib", "python tools/login.py"] - raise Exception(f'cookie.json not found. Please run `{"; ".join(commands)}` first.') - -cookies_str = "; ".join([f"{k}={v}" for k, v in cookies.items()]) - - -with open("src/config/placeholder.json", "r") as f: - placeholder = json.load(f) - def get_key(snake_str): components = snake_str.split("_") @@ -120,15 +106,15 @@ def save_cache(data): json.dump(data, f, indent=4) -# ====== Load cache ====== -for file in glob.glob("cache/*.json"): - with open(file, "r") as f: - cache = json.load(f) - data = pt.__dict__[cache["type"]].from_json(cache["raw"]) - rate = match_rate(data.to_dict(), json.loads(cache["raw"])) - logger.info(f"Match rate: {rate}") - -logger.info(f"Success: {len(glob.glob('cache/*.json'))}") +def task_callback(file): + try: + with open(file, "r") as f: + cache = json.load(f) + data = pt.__dict__[cache["type"]].from_json(cache["raw"]) + rate = match_rate(data.to_dict(), json.loads(cache["raw"])) + return f"Match rate: {rate}" + except Exception as e: + return f"Error: {e} {file}" def error_dump(e): @@ -142,90 +128,122 @@ def error_dump(e): logger.info("================================") -api_conf = pt.Configuration( - api_key={ - "ClientLanguage": "en", - "ActiveUser": "yes", - "AuthType": "OAuth2Session", - "CsrfToken": cookies["ct0"], - }, -) +if __name__ == "__main__": + if Path("cookie.json").exists(): + with open("cookie.json", "r") as f: + cookies = json.load(f) + elif TWITTER_SESSION is not None: + data = base64.b64decode(TWITTER_SESSION).decode("utf-8") + cookies = json.loads(data) + else: + commands = ["python -m pip install tweepy_authlib", "python tools/login.py"] + raise Exception( + f'cookie.json not found. Please run `{"; ".join(commands)}` first.' + ) -api_conf.access_token = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" + cookies_str = "; ".join([f"{k}={v}" for k, v in cookies.items()]) -api_client = pt.ApiClient(configuration=api_conf, cookie=cookies_str) -api_client.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36" + with open("src/config/placeholder.json", "r") as f: + placeholder = json.load(f) -error_count = 0 + with concurrent.futures.ProcessPoolExecutor() as executor: + tasks = [executor.submit(task_callback, x) for x in glob.glob("cache/*.json")] + for task in concurrent.futures.as_completed(tasks): + logger.info(task.result()) -for x in [pt.DefaultApi, pt.TweetApi, pt.UserApi, pt.UsersApi, pt.UserListApi]: - for props, fn in x.__dict__.items(): - if not callable(fn): - continue - if props.startswith("__") or not props.endswith("_with_http_info"): - continue + logger.info(f"Success: {len(glob.glob('cache/*.json'))}") - key = get_key(props[:-15]) - cursor_list = set([None]) - cursor_history = set() + api_conf = pt.Configuration( + api_key={ + "ClientLanguage": "en", + "ActiveUser": "yes", + "AuthType": "OAuth2Session", + "CsrfToken": cookies["ct0"], + }, + ) - try: - for _ in range(CUESOR_TEST_COUNT): - cursor = cursor_list.pop() - cursor_history.add(cursor) - logger.info(f"Try: {key} {cursor}") + api_conf.access_token = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" - kwargs = get_kwargs(key, {} if cursor is None else {"cursor": cursor}) - res: pt.ApiResponse = getattr(x(api_client), props)(**kwargs) - data = res.data.to_dict() + api_client = pt.ApiClient(configuration=api_conf, cookie=cookies_str) + api_client.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36" - save_cache({"raw": res.raw_data, "type": res.data.__class__.__name__}) + error_count = 0 - new_cursor = set(get_cursor(data, find_cursor)) - cursor_history - cursor_list.update(new_cursor) + for x in [pt.DefaultApi, pt.TweetApi, pt.UserApi, pt.UsersApi, pt.UserListApi]: + for props, fn in x.__dict__.items(): + if not callable(fn): + continue + if props.startswith("__") or not props.endswith("_with_http_info"): + continue - rate = match_rate(data, json.loads(res.raw_data)) - logger.info(f"Match rate: {rate}") + key = get_key(props[:-15]) + cursor_list = set([None]) + cursor_history = set() - if data.get("errors") is not None: - logger.error(data) - error_count += 1 + try: + for _ in range(CUESOR_TEST_COUNT): + cursor = cursor_list.pop() + cursor_history.add(cursor) + logger.info(f"Try: {key} {cursor}") - if len(cursor_list) == 0: - break - time.sleep(SLEEP_TIME) + kwargs = get_kwargs( + key, {} if cursor is None else {"cursor": cursor} + ) + res: pt.ApiResponse = getattr(x(api_client), props)(**kwargs) + data = res.data.to_dict() - except Exception as e: - error_dump(e) - error_count += 1 + save_cache( + { + "raw": res.raw_data, + "type": res.data.__class__.__name__, + } + ) + new_cursor = set(get_cursor(data, find_cursor)) - cursor_history + cursor_list.update(new_cursor) -try: - logger.info(f"Try: Self UserByScreenName Test") - kwargs = get_kwargs("UserByScreenName", {"screen_name": "a810810931931"}) - res = pt.UserApi(api_client).get_user_by_screen_name_with_http_info(**kwargs) - data = res.data.to_dict() + rate = match_rate(data, json.loads(res.raw_data)) + logger.info(f"Match rate: {rate}") - rate = match_rate(data, json.loads(res.raw_data)) - logger.info(f"Match rate: {rate}") - if not data["data"]["user"]["result"]["legacy"]["screen_name"] == "a810810931931": - raise Exception("UserByScreenName failed") -except Exception as e: - error_dump(e) - error_count += 1 + if data.get("errors") is not None: + logger.error(data) + error_count += 1 -try: - logger.info(f"Try: Self UserTweets Test") - kwargs = get_kwargs("UserTweets", {"userId": "1180389371481976833"}) - res = pt.TweetApi(api_client).get_user_tweets_with_http_info(**kwargs) - data = res.data.to_dict() + if len(cursor_list) == 0: + break + time.sleep(SLEEP_TIME) - rate = match_rate(data, json.loads(res.raw_data)) - logger.info(f"Match rate: {rate}") + except Exception as e: + error_dump(e) + error_count += 1 -except Exception as e: - error_dump(e) - error_count += 1 + try: + logger.info(f"Try: Self UserByScreenName Test") + kwargs = get_kwargs("UserByScreenName", {"screen_name": "a810810931931"}) + res = pt.UserApi(api_client).get_user_by_screen_name_with_http_info(**kwargs) + data = res.data.to_dict() -if error_count > 0: - exit(1) + rate = match_rate(data, json.loads(res.raw_data)) + logger.info(f"Match rate: {rate}") + screen_name = data["data"]["user"]["result"]["legacy"]["screen_name"] + if not screen_name == "a810810931931": + raise Exception("UserByScreenName failed") + except Exception as e: + error_dump(e) + error_count += 1 + + try: + logger.info(f"Try: Self UserTweets Test") + kwargs = get_kwargs("UserTweets", {"userId": "1180389371481976833"}) + res = pt.TweetApi(api_client).get_user_tweets_with_http_info(**kwargs) + data = res.data.to_dict() + + rate = match_rate(data, json.loads(res.raw_data)) + logger.info(f"Match rate: {rate}") + + except Exception as e: + error_dump(e) + error_count += 1 + + if error_count > 0: + exit(1) From 0536aba4f4c721bcfce9614ccee06a306f184631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 15 Sep 2023 12:19:21 +0900 Subject: [PATCH 20/24] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/content.yaml | 10 +++++----- src/openapi/schemas/instruction.yaml | 5 +++-- test/python/test_serialize.py | 24 ++++++++++++++++++------ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/openapi/schemas/content.yaml b/src/openapi/schemas/content.yaml index fdd72fd..184e274 100644 --- a/src/openapi/schemas/content.yaml +++ b/src/openapi/schemas/content.yaml @@ -40,8 +40,7 @@ components: itemContent: $ref: "#/components/schemas/ItemContentUnion" clientEventInfo: - type: object - additionalProperties: true # todo + $ref: "#/components/schemas/ClientEventInfo" feedbackInfo: type: object additionalProperties: true # todo @@ -113,8 +112,7 @@ components: - "itemContent" properties: clientEventInfo: - type: object - additionalProperties: true # todo + $ref: "#/components/schemas/ClientEventInfo" itemContent: $ref: "#/components/schemas/ItemContentUnion" @@ -229,4 +227,6 @@ components: type: string # enum half_cover element: type: string - pattern: "^(january|february|march|april|may|june|july|august|september|october|november|december)-[0-9]{4}-([a-z]-)+[a-z]+$" # august-2023-privacy-prompt-candidate + # august-2023-privacy-prompt-candidate + pattern: "(([a-z]+|[0-9]+|[0-9a-f]+)(-|$))+" + # pattern: "^(january|february|march|april|may|june|july|august|september|october|november|december)-[0-9]{4}-([a-z]-)+[a-z]+$" diff --git a/src/openapi/schemas/instruction.yaml b/src/openapi/schemas/instruction.yaml index 73bb9e6..25a3def 100644 --- a/src/openapi/schemas/instruction.yaml +++ b/src/openapi/schemas/instruction.yaml @@ -267,7 +267,7 @@ components: items: $ref: "#/components/schemas/Callback" clientEventInfo: - $ref: "#/components/schemas/ClientEventInfo" + $ref: "#/components/schemas/CtaClientEventInfo" buttonStyle: type: string enum: ["Primary"] @@ -286,9 +286,10 @@ components: properties: endpoint: type: string + format: uri # pattern: '^/1\.1/[a-z]+/[a-z]+\.json\?[a-z_]+=[a-z0-9-]+(&[a-z_]+=[a-z0-9-]+)+?$' #/1.1/onboarding/fatigue.json?{params} - ClientEventInfo: + CtaClientEventInfo: required: - action properties: diff --git a/test/python/test_serialize.py b/test/python/test_serialize.py index bd7f518..b619e6c 100644 --- a/test/python/test_serialize.py +++ b/test/python/test_serialize.py @@ -8,6 +8,7 @@ import time import glob import aenum import concurrent.futures +import traceback logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s") @@ -106,21 +107,23 @@ def save_cache(data): json.dump(data, f, indent=4) -def task_callback(file): +def task_callback(file, thread=True): try: with open(file, "r") as f: cache = json.load(f) data = pt.__dict__[cache["type"]].from_json(cache["raw"]) rate = match_rate(data.to_dict(), json.loads(cache["raw"])) - return f"Match rate: {rate}" + return {rate}, {file} except Exception as e: - return f"Error: {e} {file}" + if thread: + return 0, {file} + else: + raise def error_dump(e): if ERROR_UNCATCHED: raise - import traceback logger.error("==========[STACK TRACE]==========") for trace in traceback.format_exc().split("\n"): @@ -146,12 +149,21 @@ if __name__ == "__main__": with open("src/config/placeholder.json", "r") as f: placeholder = json.load(f) + fail = [] with concurrent.futures.ProcessPoolExecutor() as executor: tasks = [executor.submit(task_callback, x) for x in glob.glob("cache/*.json")] for task in concurrent.futures.as_completed(tasks): - logger.info(task.result()) + rate, file = [list(x).pop() for x in task.result()] + print(rate, file) + if rate < 1: + fail.append(file) + logger.info(f"Match rate: {rate}") - logger.info(f"Success: {len(glob.glob('cache/*.json'))}") + logger.info(f"Fail: {len(fail)} / {len(glob.glob('cache/*.json'))}") + + for file in fail: + task_callback(file, thread=False) + logger.info(f"Match rate: {rate}") api_conf = pt.Configuration( api_key={ From bbbec5569f85f74894dd8f6a1930e2fc70c207a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Fri, 15 Sep 2023 19:16:48 +0900 Subject: [PATCH 21/24] update test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/content.yaml | 11 ++++++----- test/python/test_serialize.py | 7 +++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/openapi/schemas/content.yaml b/src/openapi/schemas/content.yaml index 184e274..b2a5ed1 100644 --- a/src/openapi/schemas/content.yaml +++ b/src/openapi/schemas/content.yaml @@ -219,14 +219,15 @@ components: additionalProperties: true # todo ClientEventInfo: - required: - - component - - elemen properties: component: - type: string # enum half_cover + # enum half_cover + type: string element: type: string # august-2023-privacy-prompt-candidate - pattern: "(([a-z]+|[0-9]+|[0-9a-f]+)(-|$))+" + # pattern: "(([a-z]+|[0-9]+|[0-9a-f]+)(-|$))+" # pattern: "^(january|february|march|april|may|june|july|august|september|october|november|december)-[0-9]{4}-([a-z]-)+[a-z]+$" + details: + type: object + additionalProperties: true # todo diff --git a/test/python/test_serialize.py b/test/python/test_serialize.py index b619e6c..4968856 100644 --- a/test/python/test_serialize.py +++ b/test/python/test_serialize.py @@ -113,10 +113,10 @@ def task_callback(file, thread=True): cache = json.load(f) data = pt.__dict__[cache["type"]].from_json(cache["raw"]) rate = match_rate(data.to_dict(), json.loads(cache["raw"])) - return {rate}, {file} + return rate, file except Exception as e: if thread: - return 0, {file} + return 0, file else: raise @@ -153,8 +153,7 @@ if __name__ == "__main__": with concurrent.futures.ProcessPoolExecutor() as executor: tasks = [executor.submit(task_callback, x) for x in glob.glob("cache/*.json")] for task in concurrent.futures.as_completed(tasks): - rate, file = [list(x).pop() for x in task.result()] - print(rate, file) + rate, file = task.result() if rate < 1: fail.append(file) logger.info(f"Match rate: {rate}") From 8bf6925a0f62b177563b8535e20649f91b8cddf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Sat, 16 Sep 2023 22:03:22 +0900 Subject: [PATCH 22/24] fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/tweet.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/openapi/schemas/tweet.yaml b/src/openapi/schemas/tweet.yaml index 1fafda0..c7d57ee 100644 --- a/src/openapi/schemas/tweet.yaml +++ b/src/openapi/schemas/tweet.yaml @@ -38,11 +38,12 @@ components: Tweet: required: - "rest_id" - - "core" + # If the tweet has been edited, this property does not exist. + # - "core" - "edit_control" - "is_translatable" - - "source" - - "legacy" + # - "source" + # - "legacy" - "views" properties: From 1d14127b79e25776e9ae226b685fccc6dc16d4ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Sun, 17 Sep 2023 03:07:32 +0900 Subject: [PATCH 23/24] fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/content.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/openapi/schemas/content.yaml b/src/openapi/schemas/content.yaml index b2a5ed1..aa2e511 100644 --- a/src/openapi/schemas/content.yaml +++ b/src/openapi/schemas/content.yaml @@ -189,7 +189,6 @@ components: ItemResult: required: - - "result" properties: __typename: $ref: "./typename.yaml#/components/schemas/TypeName" # null | TimelineTweet From bd2f821d553009111ed73c91d132cb4046e3e620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81?= Date: Sun, 17 Sep 2023 03:31:41 +0900 Subject: [PATCH 24/24] fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ふぁ --- src/openapi/schemas/tweet.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openapi/schemas/tweet.yaml b/src/openapi/schemas/tweet.yaml index c7d57ee..5ae992f 100644 --- a/src/openapi/schemas/tweet.yaml +++ b/src/openapi/schemas/tweet.yaml @@ -497,8 +497,8 @@ components: type: integer extMediaAvailability: - required: - - "status" + # required: + # - "status" properties: status: type: string