{"id":124686,"url":"https://github.com/CanvaChen/chinese-llama-tokenizer","last_synced_at":"2026-06-16T16:01:21.065Z","repository":{"id":247016467,"uuid":"683670945","full_name":"CanvaChen/chinese-llama-tokenizer","owner":"CanvaChen","description":"目标：构建一个更符合语言学的小而美的 llama 分词器，支持中英日三国语言","archived":false,"fork":false,"pushed_at":"2024-06-02T05:52:46.000Z","size":1648,"stargazers_count":20,"open_issues_count":0,"forks_count":1,"subscribers_count":2,"default_branch":"main","last_synced_at":"2026-05-12T23:08:06.133Z","etag":null,"topics":["chinese","llama","tokenizer"],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/CanvaChen.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2023-08-27T10:31:58.000Z","updated_at":"2025-08-21T02:33:51.000Z","dependencies_parsed_at":"2024-11-26T01:31:24.773Z","dependency_job_id":null,"html_url":"https://github.com/CanvaChen/chinese-llama-tokenizer","commit_stats":null,"previous_names":["canvachen/chinese-llama-tokenizer"],"tags_count":2,"template":false,"template_full_name":null,"purl":"pkg:github/CanvaChen/chinese-llama-tokenizer","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/CanvaChen%2Fchinese-llama-tokenizer","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/CanvaChen%2Fchinese-llama-tokenizer/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/CanvaChen%2Fchinese-llama-tokenizer/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/CanvaChen%2Fchinese-llama-tokenizer/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/CanvaChen","download_url":"https://codeload.github.com/CanvaChen/chinese-llama-tokenizer/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/CanvaChen%2Fchinese-llama-tokenizer/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":33899697,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-05-26T15:22:16.424Z","status":"online","status_checked_at":"2026-06-04T02:00:06.755Z","response_time":64,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"owner":{"login":"CanvaChen","name":"Canva","uuid":"18375123","kind":"user","description":"Software Engineer","email":"","website":null,"location":"Fujian, China","twitter":null,"company":null,"icon_url":"https://avatars.githubusercontent.com/u/18375123?u=01f2b0961e89260e115d34bf91132602936e91bf\u0026v=4","repositories_count":1,"last_synced_at":"2024-07-06T05:12:19.491Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/CanvaChen","funding_links":[],"total_stars":0,"followers":0,"following":0,"created_at":"2024-07-06T05:12:19.932Z","updated_at":"2024-07-06T05:12:19.932Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/CanvaChen","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/CanvaChen/repositories"},"packages":[],"commits":{"id":1640185,"full_name":"CanvaChen/chinese-llama-tokenizer","default_branch":"main","total_commits":14,"total_committers":1,"total_bot_commits":0,"total_bot_committers":0,"mean_commits":14.0,"dds":0.0,"past_year_total_commits":0,"past_year_total_committers":0,"past_year_total_bot_commits":0,"past_year_total_bot_committers":0,"past_year_mean_commits":0.0,"past_year_dds":0.0,"last_synced_at":"2026-05-12T23:04:53.087Z","last_synced_commit":"9f3bbc9ec62381748851d47f38139378d00a66fc","created_at":"2024-07-25T15:36:09.870Z","updated_at":"2026-05-12T23:04:53.072Z","committers":[{"name":"Canva","email":"canvachen@foxmail.com","login":"CanvaChen","count":14}],"past_year_committers":[],"commits_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub/repositories/CanvaChen%2Fchinese-llama-tokenizer/commits","host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2026-06-04T00:00:08.235Z","repositories_count":6253516,"commits_count":883947103,"contributors_count":35042896,"owners_count":1162753,"icon_url":"https://github.com/github.png","host_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub/repositories"}},"issues":{"table":{"full_name":"CanvaChen/chinese-llama-tokenizer","html_url":"https://github.com/CanvaChen/chinese-llama-tokenizer","last_synced_at":"2025-08-31T16:43:58.626Z","status":"error","issues_count":1,"pull_requests_count":0,"avg_time_to_close_issue":223847.0,"avg_time_to_close_pull_request":null,"issues_closed_count":1,"pull_requests_closed_count":0,"pull_request_authors_count":0,"issue_authors_count":1,"avg_comments_per_issue":2.0,"avg_comments_per_pull_request":null,"merged_pull_requests_count":0,"bot_issues_count":0,"bot_pull_requests_count":0,"past_year_issues_count":1,"past_year_pull_requests_count":0,"past_year_avg_time_to_close_issue":223847.0,"past_year_avg_time_to_close_pull_request":null,"past_year_issues_closed_count":1,"past_year_pull_requests_closed_count":0,"past_year_pull_request_authors_count":0,"past_year_issue_authors_count":1,"past_year_avg_comments_per_issue":2.0,"past_year_avg_comments_per_pull_request":null,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":0,"past_year_merged_pull_requests_count":0,"created_at":"2024-07-25T15:36:10.762Z","updated_at":"2025-09-02T02:41:31.070Z","repository_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/CanvaChen%2Fchinese-llama-tokenizer","issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/CanvaChen%2Fchinese-llama-tokenizer/issues","issue_labels_count":{"table":{}},"pull_request_labels_count":{"table":{}},"issue_author_associations_count":{"table":{"NONE":1}},"pull_request_author_associations_count":{"table":{}},"issue_authors":{"table":{"xinpengzz":1}},"pull_request_authors":{"table":{}},"host":{"table":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2026-06-04T00:00:10.470Z","repositories_count":14753989,"issues_count":33341289,"pull_requests_count":110007176,"authors_count":11291537,"icon_url":"https://github.com/github.png","host_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories","owners_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/owners","authors_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors"}},"past_year_issue_labels_count":{"table":{}},"past_year_pull_request_labels_count":{"table":{}},"past_year_issue_author_associations_count":{"table":{}},"past_year_pull_request_author_associations_count":{"table":{}},"past_year_issue_authors":{"table":{}},"past_year_pull_request_authors":{"table":{}},"maintainers":[],"active_maintainers":[]}},"events":{"total":{"WatchEvent":5},"last_year":{"WatchEvent":1}},"keywords":["chinese","llama","tokenizer"],"dependencies":[{"ecosystem":"pypi","filepath":"requirements.txt","sha":null,"kind":"manifest","created_at":"2024-07-06T05:31:48.325Z","updated_at":"2024-07-06T05:31:48.325Z","repository_link":"https://github.com/CanvaChen/chinese-llama-tokenizer/blob/main/requirements.txt","dependencies":[{"id":18920760249,"package_name":"transformers","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18920760250,"package_name":"sentencepiece","ecosystem":"pypi","requirements":"==0.2.0","direct":true,"kind":"runtime","optional":false}]}],"score":2.995732273553991,"created_at":"2024-07-25T15:26:12.770Z","updated_at":"2026-06-16T16:01:21.065Z","avatar_url":"https://github.com/CanvaChen.png","language":"Python","codemeta":null,"publiccode":null,"project_url":"https://summary.ecosyste.ms/api/v1/projects/124686","html_url":"https://summary.ecosyste.ms/projects/124686"}