[ { "id": "agents-and-tool-use", "name": "Agents and tool use", "definition": "Systems that plan, call tools and act over multi-step tasks.", "compute_anchor": "mini-SWE-agent reached 65% on SWE-bench Verified in 100 lines of Python; compute is mainly inference on existing models.", "data_anchor": "SWE-bench Verified 500 human-filtered instances; GAIA 466 questions.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): GAIA has 6 authors.", "time_horizon": "Fast moving; METR task horizon doubles about every 7 months, under 3 months on SWE-bench Verified.", "shelf_life_signals": [ "GAIA: GPT-4 with plugins 15% at release (Nov 2023)", "SWE-bench Verified: 65% with a 100-line agent by Jul 2025", "METR horizon doubling about 7 months" ], "exemplars": [ "GAIA", "SWE-bench", "Nemotron agent models", "mini-SWE-agent" ], "observed_actors": [ "Frontier labs", "nvidia", "Academic benchmark teams", "anthropic" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs and NVIDIA (Nemotron 3 Nano, Super, Ultra for sub-agents) are observed building agent models.", "b": "A 100-line agent reached 65% on SWE-bench Verified, so agent systems need little training compute.", "c": "Academic benchmark teams are observed and GAIA was built by 6 authors.", "d": "A 100-line agent on existing models fits proof-of-concept scale, but the 7-month horizon doubling gives short shelf life." }, "pipeline_role": "a ships the agent-capable models; b and c build agents and agent benchmarks on them; d packages agents into proofs of concept and value, which age fast.", "t11_topic_number": 5, "sources": [ "https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/", "https://arxiv.org/abs/2311.12983", "https://www.swebench.com/", "https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/", "https://www.anthropic.com/research/agentic-misalignment" ], "unverified": [], "papers": [ { "title": "ReAct: Synergizing Reasoning and Acting in Language Models", "authors": "Shunyu Yao et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2210.03629", "role": "founding" }, { "title": "Toolformer: Language Models Can Teach Themselves to Use Tools", "authors": "Timo Schick et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2302.04761", "role": "founding" }, { "title": "GAIA: a benchmark for General AI Assistants", "authors": "Grégoire Mialon et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2311.12983", "role": "state of the art" }, { "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "authors": "Carlos E. Jimenez et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2310.06770", "role": "state of the art" }, { "title": "The Rise and Potential of Large Language Model Based Agents: A Survey", "authors": "Zhiheng Xi et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2309.07864", "role": "survey" } ], "approved": true }, { "id": "alignment-and-safety-evaluations", "name": "Alignment and safety evaluations (model behaviour)", "definition": "Stress tests for misaligned or harmful agent behaviour.", "compute_anchor": "Mainly API access and scenario design, not training compute (inferred in the notes).", "data_anchor": "Simulated scenarios; 16 models tested as autonomous email agents.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Months.", "shelf_life_signals": [], "exemplars": [ "Agentic misalignment" ], "observed_actors": [ "anthropic" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Anthropic is observed testing 16 frontier models.", "b": "The method needs API access and scenario design rather than training compute.", "c": "The method needs API access and scenario design rather than training compute.", "d": "API-based testing is affordable, but the notes give no anchor for proof-of-concept firms doing it and the horizon is months." }, "pipeline_role": "a and c publish behavioural tests; b adapts them to local context; d can apply them as deployment checks.", "t11_topic_number": 17, "sources": [ "https://www.anthropic.com/research/agentic-misalignment" ], "unverified": [], "papers": [ { "title": "Discovering Language Model Behaviors with Model-Written Evaluations", "authors": "Ethan Perez et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2212.09251", "role": "founding" }, { "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "authors": "Mantas Mazeika et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.04249", "role": "state of the art" }, { "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training", "authors": "Evan Hubinger et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2401.05566", "role": "state of the art" }, { "title": "Alignment faking in large language models", "authors": "Ryan Greenblatt et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2412.14093", "role": "state of the art" }, { "title": "Trustworthy LLMs: a Survey and Guideline for Evaluating Large Language Models' Alignment", "authors": "Yang Liu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2308.05374", "role": "survey" } ], "approved": true }, { "id": "data-curation-and-synthetic-data", "name": "Data curation and synthetic data", "definition": "Building, filtering and generating training corpora.", "compute_anchor": "Processing compute not sourced; TabPFN trained purely on synthetic data.", "data_anchor": "FineWeb 15T tokens from 96 Common Crawl snapshots; FineWeb-Edu 1.3T; Phi-3 synthetic plus filtered web; Pile of Law 256GB legal data with law-grounded filtering.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): FineWeb has 8 authors.", "time_horizon": "Months.", "shelf_life_signals": [], "exemplars": [ "FineWeb", "Phi-3", "Pile of Law", "TabPFN" ], "observed_actors": [ "hugging-face", "microsoft", "Stanford-area groups" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Hugging Face and Microsoft are observed building 15T-token and synthetic corpora.", "b": "FineWeb was built by an 8-author team, so corpus building is team-feasible, though processing compute is not sourced.", "c": "Stanford-area groups built Pile of Law (256GB), showing academic-scale curation.", "d": "Domain curation fits client work, but the months-long horizon exceeds a typical proof of concept; the notes give no proof-of-concept anchor." }, "pipeline_role": "b and c build local and domain corpora that feed a's and b's training; d contributes client domain data under proof-of-value agreements.", "t11_topic_number": 10, "sources": [ "https://arxiv.org/abs/2404.14219", "https://arxiv.org/abs/2406.17557", "https://arxiv.org/abs/2207.00220", "https://www.nature.com/articles/s41586-024-08328-6" ], "unverified": [], "papers": [ { "title": "Textbooks Are All You Need", "authors": "Suriya Gunasekar et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.11644", "role": "founding" }, { "title": "Self-Instruct: Aligning Language Models with Self-Generated Instructions", "authors": "Yizhong Wang et al.", "year": 2022, "venue": "ACL 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2212.10560", "role": "founding" }, { "title": "The FineWeb Datasets: Decanting the Web for the Finest Text Data at Scale", "authors": "Guilherme Penedo et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.17557", "role": "state of the art" }, { "title": "DataComp-LM: In search of the next generation of training sets for language models", "authors": "Jeffrey Li et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.11794", "role": "state of the art" }, { "title": "Best Practices and Lessons Learned on Synthetic Data", "authors": "Ruibo Liu et al.", "year": 2024, "venue": "COLM 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2404.07503", "role": "survey" } ], "approved": true }, { "id": "distillation-of-frontier-models", "name": "Distillation of frontier models into smaller ones", "definition": "Transferring capability from a large teacher model to a smaller one.", "compute_anchor": "Not sourced: Qwen3 states distillation from the flagship reduces the compute needed; no figures given.", "data_anchor": "Teacher outputs; R1 states large-model reasoning patterns can improve smaller models.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Weeks.", "shelf_life_signals": [], "exemplars": [ "Qwen3", "DeepSeek-R1" ], "observed_actors": [ "alibaba", "deepseek" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "Alibaba and DeepSeek distil from their own flagships, as stated in Qwen3 and R1.", "b": "The notes give no compute figure for distillation, and the observed actors own the teacher model; the notes verify no Indonesian national compute programme, BRIN facility or ITB GPU inventory (T11 Part E item 2).", "c": "The notes give no compute figure; the closest academic anchor is the s1 fine-tune (topic 4) on an open base.", "d": "The notes give nothing placing distillation at proof-of-concept scale; d would consume distilled models rather than produce them." }, "pipeline_role": "a distils its flagships into small open models that b and c fine-tune and d deploys.", "t11_topic_number": 9, "sources": [ "https://arxiv.org/abs/2505.09388", "https://arxiv.org/abs/2501.12948" ], "unverified": [], "papers": [ { "title": "Distilling the Knowledge in a Neural Network", "authors": "Geoffrey Hinton et al.", "year": 2015, "venue": "NIPS 2014 (per arXiv comment)", "url": "https://arxiv.org/abs/1503.02531", "role": "founding" }, { "title": "DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter", "authors": "Victor Sanh et al.", "year": 2019, "venue": "NeurIPS 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1910.01108", "role": "founding" }, { "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning", "authors": "DeepSeek-AI et al.", "year": 2025, "venue": "Nature volume 645, pages 633-638 (2025)", "url": "https://arxiv.org/abs/2501.12948", "role": "state of the art" }, { "title": "Orca: Progressive Learning from Complex Explanation Traces of GPT-4", "authors": "Subhabrata Mukherjee et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.02707", "role": "state of the art" }, { "title": "A Survey on Knowledge Distillation of Large Language Models", "authors": "Xiaohan Xu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.13116", "role": "survey" } ], "approved": true }, { "id": "domain-adaptation-vertical-models", "name": "Domain adaptation (vertical models)", "definition": "Training or tuning models for a sector.", "compute_anchor": "BloombergGPT 50B trained from scratch; compute not given.", "data_anchor": "BloombergGPT: 363B finance tokens plus 345B general tokens.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Months.", "shelf_life_signals": [ "GPT-4 few-shot beat BloombergGPT on FiQA sentiment (88.11 vs 75.07) and ConvFinQA (76.48 vs 43.41); BloombergGPT led on NER (60.82 vs 56.71)", "Domain pretraining overtaken by general models" ], "exemplars": [ "BloombergGPT", "GPT-4 finance study" ], "observed_actors": [ "bloomberg", "Academic evaluators" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Bloomberg is observed training a 50B domain model, and frontier general models beat it on most tasks compared.", "b": "Domain tuning on open bases fits b; from-scratch 50B training is beyond the evidence, and GPT-4 few-shot beat BloombergGPT on 3 tasks.", "c": "From-scratch training on 708B tokens exceeds academic GPUs, while PEFT-based tuning (topic 7) fits.", "d": "The shelf-life signal favours prompting or tuning general models, which suits proof-of-value work; d would not pretrain from scratch." }, "pipeline_role": "a's general models set the bar; b and c tune for sectors; d carries tuned or prompted models into sector proofs of value.", "t11_topic_number": 21, "sources": [ "https://arxiv.org/abs/2303.17564", "https://arxiv.org/html/2305.05862v2" ], "unverified": [], "papers": [ { "title": "Don't Stop Pretraining: Adapt Language Models to Domains and Tasks", "authors": "Suchin Gururangan et al.", "year": 2020, "venue": "ACL 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2004.10964", "role": "founding" }, { "title": "BloombergGPT: A Large Language Model for Finance", "authors": "Shijie Wu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2303.17564", "role": "state of the art" }, { "title": "Large Language Models Encode Clinical Knowledge", "authors": "Karan Singhal et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2212.13138", "role": "state of the art" }, { "title": "Domain Specialization as the Key to Make Large Language Models Disruptive: A Comprehensive Survey", "authors": "Chen Ling et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.18703", "role": "survey" } ], "approved": true }, { "id": "efficiency-architectures", "name": "Efficiency architectures (MoE, sparse attention, long context)", "definition": "Reducing training and inference cost per capability.", "compute_anchor": "DeepSeek sparse attention with over 50% API price cut; YaRN context extension with 10x fewer tokens and 2.5x fewer steps than prior methods; gpt-oss MoE 5.1B of 117B active.", "data_anchor": "YaRN needs 10x fewer tokens than prior methods.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Months.", "shelf_life_signals": [ "Fixed-capability inference price falls 9x to 900x per year" ], "exemplars": [ "DeepSeek-V3.2-Exp", "YaRN", "gpt-oss MoE" ], "observed_actors": [ "deepseek", "Academic groups", "openai" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "DeepSeek and OpenAI are observed shipping sparse attention and MoE.", "b": "New architectures need pretraining-scale runs, though methods like YaRN are cheap; the notes verify no Indonesian national compute programme, BRIN facility or ITB GPU inventory (T11 Part E item 2).", "c": "Academic groups produced YaRN at 10x fewer tokens than prior methods.", "d": "The notes give no proof-of-concept anchor; context extension on open models may fit, new architectures do not." }, "pipeline_role": "a and c produce efficiency methods that lower the cost of everything b and d run; d benefits mainly as a user.", "t11_topic_number": 20, "sources": [ "https://openai.com/index/introducing-gpt-oss/", "https://api-docs.deepseek.com/news/news250929", "https://arxiv.org/abs/2309.00071" ], "unverified": [], "papers": [ { "title": "Mamba: Linear-Time Sequence Modeling with Selective State Spaces", "authors": "Albert Gu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.00752", "role": "founding" }, { "title": "FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness", "authors": "Tri Dao et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2205.14135", "role": "founding" }, { "title": "Transformers are SSMs: Generalized Models and Efficient Algorithms Through Structured State Space Duality", "authors": "Tri Dao et al.", "year": 2024, "venue": "ICML 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2405.21060", "role": "state of the art" }, { "title": "Mixtral of Experts", "authors": "Albert Q. Jiang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2401.04088", "role": "state of the art" }, { "title": "Efficient Transformers: A Survey", "authors": "Yi Tay et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2009.06732", "role": "survey" } ], "approved": true }, { "id": "evaluation-and-benchmark-building", "name": "Evaluation and benchmark building", "definition": "Designing test sets that measure capability or risk.", "compute_anchor": "Mainly expert human effort and API inference, not training compute (inferred in the notes; no cost figures).", "data_anchor": "HLE 2,500 questions; GAIA 466 questions; SWE-bench Verified 500 human-filtered instances.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): HLE about 1,000 authors; GAIA 6 authors.", "time_horizon": "Months to build; months to saturate.", "shelf_life_signals": [ "HLE built because LLMs exceed 90% on MMLU", "Agent benchmarks moving fast", "ARC-AGI-1 saturation leading to ARC-AGI-2 (unverified, page fetch failed)" ], "exemplars": [ "HLE", "GAIA", "SWE-bench Verified" ], "observed_actors": [ "Nonprofits", "Frontier labs", "Academia" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs are observed (SWE-bench Verified built with OpenAI).", "b": "Benchmarks are built with human effort and API inference, which b can fund without training compute.", "c": "GAIA was built by 6 authors, an academic-scale team.", "d": "Small domain test sets need human effort and API calls only, fitting proof-of-value measurement, though the notes give no direct proof-of-concept anchor." }, "pipeline_role": "c and b build benchmarks (including local-language ones) that grade a's releases; d uses evaluation to show proof of value to clients.", "t11_topic_number": 12, "sources": [ "https://arxiv.org/abs/2311.12983", "https://www.swebench.com/", "https://arxiv.org/abs/2501.14249" ], "unverified": [ "shelf_life_signals" ], "papers": [ { "title": "Measuring Massive Multitask Language Understanding", "authors": "Dan Hendrycks et al.", "year": 2020, "venue": "ICLR 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2009.03300", "role": "founding" }, { "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "authors": "Aarohi Srivastava et al.", "year": 2022, "venue": "Transactions on Machine Learning Research, May/2022, https://openreview.net/foru", "url": "https://arxiv.org/abs/2206.04615", "role": "founding" }, { "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference", "authors": "Wei-Lin Chiang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.04132", "role": "state of the art" }, { "title": "Holistic Evaluation of Language Models", "authors": "Percy Liang et al.", "year": 2022, "venue": "Published in Transactions on Machine Learning Research (TMLR), 2023", "url": "https://arxiv.org/abs/2211.09110", "role": "state of the art" }, { "title": "Evaluating Large Language Models: A Comprehensive Survey", "authors": "Zishan Guo et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2310.19736", "role": "survey" } ], "approved": true }, { "id": "frontier-pretraining-scaling", "name": "Frontier pretraining scaling", "definition": "Training a new base model at or near the top-5 compute of its release date (Epoch's frontier definition).", "compute_anchor": "1e25 to 1e26 FLOP; a 1e25 run costs tens of millions of dollars; first 1e26 FLOP model Grok-3 (Feb 2025); frontier compute grows 4 to 5x per year; final-run cost grows about 2.4x per year, projected above $1B by 2027; DeepSeek-V3 final run 2.788M H800 GPU hours, $5.576M, excluding ablations.", "data_anchor": "DeepSeek-V3 14.8T tokens; GLM-4.5 23T tokens.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes. R&D staff are 29 to 49% of development cost for selected frontier models.", "time_horizon": "Months per run; 4 to 5x per year compute growth.", "shelf_life_signals": [ "Open models lag closed by about 14 months weighted", "Final-run figures understate total cost because ablations and staff are excluded" ], "exemplars": [ "Grok-3", "DeepSeek-V3", "GLM-4.5" ], "observed_actors": [ "spacexai", "deepseek", "zhipu-zai", "openai", "google-deepmind", "meta", "bytedance", "huawei", "12 developers with a 1e25 FLOP model by June 2025" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "not feasible", "tier_d": "not feasible", "tier_rationale": { "a": "Observed actors are frontier labs, with 12 developers holding a 1e25 FLOP model by June 2025 at tens of millions of dollars per run.", "b": "The cheapest anchor is DeepSeek-V3 at $5.576M for the final run alone, and the notes verify no Indonesian national compute programme, BRIN facility or ITB GPU inventory (T11 Part E item 2).", "c": "Academic teams typically have 1 to 8 GPUs for days, against 2.788M H800 GPU hours for the cheapest frontier final run.", "d": "The notes give no anchor placing proof-of-concept firms anywhere near 1e25 FLOP runs; their compute scale is not described." }, "pipeline_role": "a produces frontier base models and, when released as open weights, the substrate that b, c and d adapt; the roughly 14-month open lag sets how stale that substrate is.", "t11_topic_number": 1, "sources": [ "https://epoch.ai/blog/training-compute-of-frontier-ai-models-grows-by-4-5x-per-year", "https://epoch.ai/blog/how-much-does-it-cost-to-train-frontier-ai-models", "https://epoch.ai/data-insights/models-over-1e25-flop", "https://epoch.ai/tools/model-counts", "https://epoch.ai/data/notable-ai-models", "https://arxiv.org/html/2412.19437v1", "https://arxiv.org/abs/2508.06471", "https://epoch.ai/blog/open-models-report" ], "unverified": [], "papers": [ { "title": "Scaling Laws for Neural Language Models", "authors": "Jared Kaplan et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2001.08361", "role": "founding" }, { "title": "Training Compute-Optimal Large Language Models", "authors": "Jordan Hoffmann et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2203.15556", "role": "founding" }, { "title": "The Llama 3 Herd of Models", "authors": "Aaron Grattafiori et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2407.21783", "role": "state of the art" }, { "title": "A Survey of Large Language Models", "authors": "Wayne Xin Zhao et al.", "year": 2023, "venue": "Frontiers of Computer Science, Volume 20, Issue 12, Article 2012627 (2026)", "url": "https://arxiv.org/abs/2303.18223", "role": "survey" } ], "approved": true }, { "id": "guardrails-and-safety-classifiers", "name": "Guardrails and safety classifiers", "definition": "Input and output moderation models.", "compute_anchor": "Llama Guard: Llama2-7B instruction-tuned on a low-volume dataset; compute not given.", "data_anchor": "Low-volume dataset labelled against a risk taxonomy.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Weeks.", "shelf_life_signals": [], "exemplars": [ "Llama Guard", "Nemotron Safety models" ], "observed_actors": [ "meta", "nvidia" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Meta and NVIDIA are observed shipping guard models.", "b": "A 7B model tuned on a low-volume dataset is within reach for a lab that trained a 70B model.", "c": "Tuning a 7B model on a small dataset fits academic GPUs when combined with QLoRA-type methods (topic 7).", "d": "A weeks-long 7B fine-tune on a small dataset fits proof-of-concept scale; the notes give no direct proof-of-concept anchor." }, "pipeline_role": "a releases guard models; b and c tune them to local languages and norms; d puts them in front of client deployments.", "t11_topic_number": 14, "sources": [ "https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/", "https://arxiv.org/abs/2312.06674" ], "unverified": [], "papers": [ { "title": "Llama Guard: LLM-based Input-Output Safeguard for Human-AI Conversations", "authors": "Hakan Inan et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.06674", "role": "founding" }, { "title": "Constitutional Classifiers: Defending against Universal Jailbreaks across Thousands of Hours of Red Teaming", "authors": "Mrinank Sharma et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2501.18837", "role": "state of the art" }, { "title": "WildGuard: Open One-Stop Moderation Tools for Safety Risks, Jailbreaks, and Refusals of LLMs", "authors": "Seungju Han et al.", "year": 2024, "venue": "NeurIPS 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2406.18495", "role": "state of the art" }, { "title": "ShieldGemma: Generative AI Content Moderation Based on Gemma", "authors": "Wenjun Zeng et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2407.21772", "role": "state of the art" }, { "title": "NeMo Guardrails: A Toolkit for Controllable and Safe LLM Applications with Programmable Rails", "authors": "Traian Rebedea et al.", "year": 2023, "venue": "EMNLP 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2310.10501", "role": "state of the art" } ], "approved": true }, { "id": "interpretability-small-open-models", "name": "Interpretability on small open models", "definition": "Mechanistic interpretability methods applied to 1B to 9B open models.", "compute_anchor": "Hardware requirements for circuit-tracer not stated; Gemma Scope SAEs on all layers of Gemma 2 2B and 9B, select layers of 27B.", "data_anchor": "Open weights of Gemma-2-2B, Llama-3.2-1B, Gemma 2 9B.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Weeks to months.", "shelf_life_signals": [ "Google DeepMind deprioritised fundamental SAE research in March 2025 after SAE probes underperformed linear probes out of distribution" ], "exemplars": [ "circuit-tracer", "Gemma Scope" ], "observed_actors": [ "anthropic", "google-deepmind", "Open community (Neuronpedia)" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Anthropic and Google DeepMind are observed and released the tools.", "b": "Tools are open-sourced for 1B to 2B models, though hardware needs are not stated in the notes.", "c": "Open tools on 1B to 9B models suit academic work; hardware needs are not stated in the notes.", "d": "The notes give no anchor linking this to proof-of-concept or proof-of-value work, and SAE research was deprioritised by Google DeepMind." }, "pipeline_role": "a open-sources tools (circuit-tracer, Gemma Scope); c and the open community apply them to small models.", "t11_topic_number": 16, "sources": [ "https://anthropic.com/research/open-source-circuit-tracing", "https://arxiv.org/abs/2408.05147", "https://deepmindsafetyresearch.medium.com/negative-results-for-sparse-autoencoders-on-downstream-tasks-and-deprioritising-sae-research-6cadcfc125b9" ], "unverified": [], "papers": [ { "title": "TinyStories: How Small Can Language Models Be and Still Speak Coherent English?", "authors": "Ronen Eldan et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.07759", "role": "founding" }, { "title": "Sparse Autoencoders Find Highly Interpretable Features in Language Models", "authors": "Hoagy Cunningham et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2309.08600", "role": "founding" }, { "title": "OLMo: Accelerating the Science of Language Models", "authors": "Dirk Groeneveld et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.00838", "role": "state of the art" }, { "title": "Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling", "authors": "Stella Biderman et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2304.01373", "role": "state of the art" } ], "approved": true }, { "id": "low-resource-regional-language-resources", "name": "Low-resource and regional language resources", "definition": "Datasets, benchmarks and adaptation for under-served languages.", "compute_anchor": "Data effort dominates compute; compute not sourced.", "data_anchor": "NusaX: 10 Indonesian local languages; NusaCrowd: 137 datasets, 118 loaders, first multilingual ASR benchmark for Indonesian languages.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Months to years.", "shelf_life_signals": [ "Qwen3 went from 29 to 119 languages and dialects, frontier lab overlap with regional coverage" ], "exemplars": [ "NusaX", "NusaCrowd", "SEA-LION" ], "observed_actors": [ "Indonesian NLP community", "ai-singapore" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Qwen3's growth to 119 languages shows frontier labs can cover regional languages.", "b": "Sahabat-AI covers five Indonesian languages, and the work is data-heavy rather than compute-heavy.", "c": "The Indonesian NLP community built NusaX and NusaCrowd, which are data efforts rather than compute efforts.", "d": "Months-to-years data work is long for a proof of concept; the notes give no proof-of-concept anchor." }, "pipeline_role": "c and b build local datasets and benchmarks that a and b train on and d uses to show local-language value.", "t11_topic_number": 11, "sources": [ "https://arxiv.org/abs/2505.09388", "https://www.gotocompany.com/en/news/press/sahabat-ai-gets-smarter-indosat-and-goto-launch-new-70-billion-parameter-model-with-multilingual-chat-service", "https://arxiv.org/abs/2504.05747", "https://arxiv.org/abs/2205.15960", "https://arxiv.org/abs/2212.09648" ], "unverified": [ "observed_actors" ], "papers": [ { "title": "IndoNLU: Benchmark and Resources for Evaluating Indonesian Natural Language Understanding", "authors": "Bryan Wilie et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2009.05387", "role": "founding" }, { "title": "SEA-LION: Southeast Asian Languages in One Network", "authors": "Raymond Ng et al.", "year": 2025, "venue": "ACL 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2504.05747", "role": "state of the art" }, { "title": "Aya Dataset: An Open-Access Collection for Multilingual Instruction Tuning", "authors": "Shivalika Singh et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.06619", "role": "state of the art" }, { "title": "Aya Model: An Instruction Finetuned Open-Access Multilingual Language Model", "authors": "Ahmet Üstün et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.07827", "role": "state of the art" }, { "title": "NusaCrowd: Open Source Initiative for Indonesian NLP Resources", "authors": "Samuel Cahyawijaya et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2212.09648", "role": "state of the art" } ], "approved": true }, { "id": "mechanistic-interpretability-frontier-scale", "name": "Mechanistic interpretability, frontier scale", "definition": "Reverse-engineering the internals of production models.", "compute_anchor": "Claude 3.5 Haiku with a cross-layer transcoder of about 30M features; hardware not given.", "data_anchor": "Production model weights, which only the developer has.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Multi-year agenda.", "shelf_life_signals": [ "Satisfying insight on only about a quarter of prompts tried" ], "exemplars": [ "On the Biology of a Large Language Model" ], "observed_actors": [ "anthropic" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "not feasible", "tier_d": "not feasible", "tier_rationale": { "a": "Anthropic is the only observed actor and has the production weights.", "b": "The work requires production weights that only the developer holds.", "c": "The work requires production weights that only the developer holds.", "d": "The work requires production weights that only the developer holds." }, "pipeline_role": "a produces methods and tools, some later open-sourced for small models (topic 16), which pass to b and c.", "t11_topic_number": 15, "sources": [ "https://transformer-circuits.pub/2025/attribution-graphs/biology.html" ], "unverified": [], "papers": [ { "title": "Toy Models of Superposition", "authors": "Nelson Elhage et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2209.10652", "role": "founding" }, { "title": "Scaling and evaluating sparse autoencoders", "authors": "Leo Gao et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.04093", "role": "state of the art" }, { "title": "Sparse Feature Circuits: Discovering and Editing Interpretable Causal Graphs in Language Models", "authors": "Samuel Marks et al.", "year": 2024, "venue": "International Conference on Learning Representations, 2025", "url": "https://arxiv.org/abs/2403.19647", "role": "state of the art" }, { "title": "Gemma Scope: Open Sparse Autoencoders Everywhere All At Once on Gemma 2", "authors": "Tom Lieberum et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2408.05147", "role": "state of the art" }, { "title": "Mechanistic Interpretability for AI Safety -- A Review", "authors": "Leonard Bereska et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2404.14082", "role": "survey" } ], "approved": true }, { "id": "open-weights-as-research-substrate", "name": "Open-weight frontier releases as research substrate", "definition": "Using released weights as the base for downstream research.", "compute_anchor": "gpt-oss-120b runs on one 80GB GPU under Apache 2.0; Magistral Small 24B Apache 2.0.", "data_anchor": "Nemotron publishes data, techniques and weights under the NVIDIA Open Model License; Qwen3 0.6B to 235B.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Continuous.", "shelf_life_signals": [ "Open models lag closed by about 14 months weighted" ], "exemplars": [ "gpt-oss", "Nemotron", "Magistral Small", "Qwen3" ], "observed_actors": [ "openai", "nvidia", "mistral-ai", "alibaba" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "OpenAI, NVIDIA, Mistral and Alibaba are observed releasing open weights.", "b": "Sahabat-AI and SEA-LION were built on open bases.", "c": "gpt-oss-120b fits one 80GB GPU, and the compute divide paper notes academia shifting toward industry open models.", "d": "Permissive licences (Apache 2.0) and single-GPU footprints make open weights the practical base for proofs of concept." }, "pipeline_role": "The hinge of the loop: a releases open weights, b and c adapt and study them, d deploys them, with a 14-month lag behind closed models.", "t11_topic_number": 24, "sources": [ "https://arxiv.org/abs/2505.09388", "https://arxiv.org/abs/2401.02452v2", "https://epoch.ai/blog/open-models-report", "https://mistral.ai/news/magistral", "https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/", "https://openai.com/index/introducing-gpt-oss/" ], "unverified": [], "papers": [ { "title": "BLOOM: A 176B-Parameter Open-Access Multilingual Language Model", "authors": "BigScience Workshop et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2211.05100", "role": "founding" }, { "title": "OLMo: Accelerating the Science of Language Models", "authors": "Dirk Groeneveld et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.00838", "role": "state of the art" }, { "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models", "authors": "Hugo Touvron et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2307.09288", "role": "state of the art" }, { "title": "Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling", "authors": "Stella Biderman et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2304.01373", "role": "state of the art" }, { "title": "On the Societal Impact of Open Foundation Models", "authors": "Sayash Kapoor et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.07918", "role": "survey" } ], "approved": true }, { "id": "parameter-efficient-fine-tuning", "name": "Parameter-efficient fine-tuning (PEFT)", "definition": "Adapting models by training a small subset of parameters or a low-rank update.", "compute_anchor": "LoRA: about 10,000x fewer trainable parameters and about 3x less memory than full fine-tuning of GPT-3 175B; QLoRA: 65B on one 48GB GPU, Guanaco 99.3% of ChatGPT on Vicuna after 24 hours on one GPU.", "data_anchor": "Task-specific fine-tuning data; volume not given.", "team_size_proxy": "Proxy only: LoRA and QLoRA papers have small author lists (counts not given).", "time_horizon": "Days.", "shelf_life_signals": [], "exemplars": [ "LoRA", "QLoRA" ], "observed_actors": [ "Academia and industry broadly", "nvidia" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Broadly used by industry, and a 65B fine-tune fits one 48GB GPU.", "b": "QLoRA fits a 65B model on one 48GB GPU in 24 hours.", "c": "One 48GB GPU for 24 hours falls inside the academic A6000 or A100 range of 1 to 8 GPUs.", "d": "NVIDIA states SLM fine-tuning needs a few GPU-hours, within a proof-of-concept budget." }, "pipeline_role": "Main adaptation path: a's open weights are tuned by b and c with LoRA or QLoRA, and d uses the same path for client-specific proofs of concept.", "t11_topic_number": 7, "sources": [ "https://arxiv.org/html/2410.23261v1", "https://arxiv.org/html/2506.02153v1", "https://arxiv.org/abs/2106.09685", "https://arxiv.org/abs/2305.14314" ], "unverified": [], "papers": [ { "title": "LoRA: Low-Rank Adaptation of Large Language Models", "authors": "Edward J. Hu et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2106.09685", "role": "founding" }, { "title": "Parameter-Efficient Transfer Learning for NLP", "authors": "Neil Houlsby et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1902.00751", "role": "founding" }, { "title": "QLoRA: Efficient Finetuning of Quantized LLMs", "authors": "Tim Dettmers et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.14314", "role": "state of the art" }, { "title": "Parameter-Efficient Fine-Tuning for Large Models: A Comprehensive Survey", "authors": "Zeyu Han et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.14608", "role": "survey" } ], "approved": true }, { "id": "quantization-and-edge-inference", "name": "Quantization and edge inference", "definition": "Lower-bit weights and on-device serving.", "compute_anchor": "AWQ: weight-only low-bit quantization with small offline calibration and no backpropagation, over 3x faster than FP16 on desktop and mobile GPUs (TinyChat); gpt-oss-20b runs in 16GB, gpt-oss-120b on one 80GB GPU.", "data_anchor": "Small offline calibration set.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Days to weeks.", "shelf_life_signals": [ "Fixed-capability inference price falls 9x to 900x per year" ], "exemplars": [ "AWQ", "TinyChat", "gpt-oss" ], "observed_actors": [ "Academic groups", "openai" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "OpenAI ships gpt-oss sized for 16GB and single 80GB GPU deployment.", "b": "AWQ needs only a small calibration set and no backpropagation.", "c": "Academic groups produced AWQ, and the method needs no training compute.", "d": "gpt-oss-20b in 16GB and over 3x speedups on desktop and mobile GPUs fit proof-of-technology work on modest hardware." }, "pipeline_role": "Takes a's open weights and makes them deployable on the hardware b, c and d actually have; d relies on it for on-premise and edge proofs of concept.", "t11_topic_number": 8, "sources": [ "https://epoch.ai/data-insights/llm-inference-price-trends", "https://openai.com/index/introducing-gpt-oss/", "https://arxiv.org/abs/2306.00978" ], "unverified": [], "papers": [ { "title": "LLM.int8(): 8-bit Matrix Multiplication for Transformers at Scale", "authors": "Tim Dettmers et al.", "year": 2022, "venue": "NeurIPS 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2208.07339", "role": "founding" }, { "title": "AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration", "authors": "Ji Lin et al.", "year": 2023, "venue": "MLSys 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2306.00978", "role": "state of the art" }, { "title": "GPTQ: Accurate Post-Training Quantization for Generative Pre-trained Transformers", "authors": "Elias Frantar et al.", "year": 2022, "venue": "ICLR 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2210.17323", "role": "state of the art" }, { "title": "SmoothQuant: Accurate and Efficient Post-Training Quantization for Large Language Models", "authors": "Guangxuan Xiao et al.", "year": 2022, "venue": "ICML 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2211.10438", "role": "state of the art" }, { "title": "A Survey of Quantization Methods for Efficient Neural Network Inference", "authors": "Amir Gholami et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2103.13630", "role": "survey" } ], "approved": true }, { "id": "reasoning-via-large-scale-rl", "name": "Reasoning via large-scale RL", "definition": "Reinforcement learning on outcomes, often verifiable math or code, to produce long reasoning traces.", "compute_anchor": "Grok 4 RL at pretraining scale on a 200,000 GPU cluster; MiniMax-M1 RL $534,700 on 512 H800 for about 3 weeks; DeepSeek-R1 RL stage $294,000 on top of an about $6M base model.", "data_anchor": "Verifiable rewards; R1 states reasoning can be incentivised through pure RL without human-labelled trajectories.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Weeks per RL run.", "shelf_life_signals": [ "o1 performance improves smoothly with train-time and test-time compute" ], "exemplars": [ "o1", "DeepSeek-R1", "Grok 4", "Magistral" ], "observed_actors": [ "openai", "spacexai", "deepseek", "minimax", "mistral-ai" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "not feasible", "tier_d": "not feasible", "tier_rationale": { "a": "OpenAI, xAI, DeepSeek, MiniMax and Mistral are observed, with Grok 4 RL at 200,000-GPU pretraining scale.", "b": "The cheapest anchors are $294,000 to $534,700 for the RL stage alone on top of a strong base (MiniMax used 512 H800 for 3 weeks), and the notes verify no Indonesian national compute programme, BRIN facility or ITB GPU inventory (T11 Part E item 2).", "c": "512 H800 for 3 weeks far exceeds the typical academic 1 to 8 GPUs, and 85% of surveyed academics have no cloud budget.", "d": "The notes give no anchor for proof-of-concept firms at hundreds of GPUs for weeks." }, "pipeline_role": "a produces reasoning models and open reasoning checkpoints (R1, Magistral Small) that b and c distil or fine-tune and d deploys.", "t11_topic_number": 3, "sources": [ "https://x.ai/news/grok-4", "https://eu.36kr.com/en/p/3471852231792260", "https://arxiv.org/abs/2506.13585", "https://arxiv.org/html/2410.23261v1", "https://openai.com/index/learning-to-reason-with-llms/", "https://mistral.ai/news/magistral", "https://arxiv.org/abs/2501.12948" ], "unverified": [], "papers": [ { "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models", "authors": "Jason Wei et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2201.11903", "role": "founding" }, { "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning", "authors": "DeepSeek-AI et al.", "year": 2025, "venue": "Nature volume 645, pages 633-638 (2025)", "url": "https://arxiv.org/abs/2501.12948", "role": "state of the art" }, { "title": "DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models", "authors": "Zhihong Shao et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.03300", "role": "state of the art" }, { "title": "Let's Verify Step by Step", "authors": "Hunter Lightman et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.20050", "role": "state of the art" }, { "title": "Towards Large Reasoning Models: A Survey of Reinforced Reasoning with Large Language Models", "authors": "Fengli Xu et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2501.09686", "role": "survey" } ], "approved": true }, { "id": "regional-national-foundation-model-training", "name": "Regional or national foundation model training (continued pretraining at 7B to 70B)", "definition": "Pretraining or continued pretraining a mid-size model to serve a national or regional language set.", "compute_anchor": "Sahabat-AI 70B trained on Indosat GPU Merdeka via Lintasarta (no hardware figures published); SEA-LION v3 built on 8B and 9B models; compute not published by either source.", "data_anchor": "Sahabat-AI covers Bahasa Indonesia, Javanese, Sundanese, Balinese, Bataknese; SEA-LION covers 11 Southeast Asian languages; token counts not published.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): SEA-LION paper has 30 authors.", "time_horizon": "Release cadence about yearly (unverified).", "shelf_life_signals": [ "Open-vs-closed lag of about 14 months bears on how long a regional model on an open base stays competitive", "Qwen3 expanded from 29 to 119 languages, a frontier lab entering regional language coverage" ], "exemplars": [ "Sahabat-AI", "SEA-LION" ], "observed_actors": [ "goto", "indosat", "lintasarta", "ai-singapore" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "Qwen3's expansion to 119 languages shows a frontier lab already covering regional languages at scale.", "b": "Sahabat-AI 70B was trained in Indonesia by GoTo and Indosat on GPU Merdeka, although no hardware or token figures were published.", "c": "Academic anchors (1 to 8 GPUs; Pythia-1B needs 18 to 41 days on 4 A100) make 7B to 9B continued pretraining a stretch and 70B out of reach.", "d": "The notes show only telco-scale or national-programme actors doing this, with no proof-of-concept-scale anchor." }, "pipeline_role": "a supplies the open base models; b continues pretraining them for national languages and passes the regional model to c for evaluation and to d for proof-of-value deployments.", "t11_topic_number": 2, "sources": [ "https://arxiv.org/abs/2505.09388", "https://arxiv.org/html/2410.23261v1", "https://epoch.ai/blog/open-models-report", "https://www.gotocompany.com/en/news/press/sahabat-ai-gets-smarter-indosat-and-goto-launch-new-70-billion-parameter-model-with-multilingual-chat-service", "https://arxiv.org/abs/2504.05747" ], "unverified": [ "time_horizon", "observed_actors" ], "papers": [ { "title": "SEA-LION: Southeast Asian Languages in One Network", "authors": "Raymond Ng et al.", "year": 2025, "venue": "ACL 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2504.05747", "role": "state of the art" }, { "title": "Sailor: Open Language Models for South-East Asia", "authors": "Longxu Dou et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2404.03608", "role": "state of the art" }, { "title": "Typhoon: Thai Large Language Models", "authors": "Kunat Pipatanakul et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.13951", "role": "state of the art" }, { "title": "SeaLLMs -- Large Language Models for Southeast Asia", "authors": "Xuan-Phi Nguyen et al.", "year": 2023, "venue": "ACL 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2312.00738", "role": "state of the art" }, { "title": "Jais and Jais-chat: Arabic-Centric Foundation and Instruction-Tuned Open Generative Large Language Models", "authors": "Neha Sengupta et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2308.16149", "role": "state of the art" } ], "approved": true }, { "id": "retrieval-augmented-generation", "name": "Retrieval-augmented generation (RAG)", "definition": "Grounding generation in retrieved documents.", "compute_anchor": "Inference-heavy, training-light; compute not sourced.", "data_anchor": "Client or domain document collections; not quantified.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Weeks for a system.", "shelf_life_signals": [], "exemplars": [ "RAG survey (Naive, Advanced, Modular RAG)", "Nemotron Retriever models" ], "observed_actors": [ "Academia", "nvidia", "Integrators" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "NVIDIA ships Nemotron Retriever models for RAG.", "b": "RAG is training-light and weeks per system; the notes give no compute anchor.", "c": "Academia is observed, and the method needs inference rather than training.", "d": "Integrators are observed and a system takes weeks, which fits proof-of-concept delivery; the notes give no compute anchor." }, "pipeline_role": "a supplies retriever and generator models; c studies methods; b and d build grounded systems on local documents for proofs of concept and value.", "t11_topic_number": 13, "sources": [ "https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/", "https://arxiv.org/abs/2312.10997" ], "unverified": [], "papers": [ { "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks", "authors": "Patrick Lewis et al.", "year": 2020, "venue": "NeurIPS 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2005.11401", "role": "founding" }, { "title": "REALM: Retrieval-Augmented Language Model Pre-Training", "authors": "Kelvin Guu et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2002.08909", "role": "founding" }, { "title": "Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection", "authors": "Akari Asai et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2310.11511", "role": "state of the art" }, { "title": "Atlas: Few-shot Learning with Retrieval Augmented Language Models", "authors": "Gautier Izacard et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2208.03299", "role": "state of the art" }, { "title": "Retrieval-Augmented Generation for Large Language Models: A Survey", "authors": "Yunfan Gao et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.10997", "role": "survey" } ], "approved": true }, { "id": "small-data-reasoning-fine-tuning", "name": "Small-data reasoning fine-tuning and distillation", "definition": "SFT on hundreds to thousands of curated reasoning traces from a stronger model.", "compute_anchor": "s1: 16 H100 for 26 minutes on Qwen2.5-32B-Instruct; Absolute Zero compute not reported.", "data_anchor": "s1: 1,000 curated examples; LIMO: about 1% of prior training data; Absolute Zero: self-play with a code executor, no external data.", "team_size_proxy": "Proxy only: s1 author institutions are Stanford, UW, AI2 and Contextual AI; author count not given in the notes.", "time_horizon": "Days to weeks.", "shelf_life_signals": [ "All three exemplars depend on an existing strong open base model, so they track frontier open releases (topic 24)" ], "exemplars": [ "s1", "LIMO", "Absolute Zero" ], "observed_actors": [ "stanford-university", "university-of-washington", "allen-ai", "contextual-ai", "Universities and nonprofits" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs hold the strong base models these methods depend on, so the topic is within their reach.", "b": "s1 needed 16 H100 for 26 minutes and 1,000 examples, far below the Sahabat-AI training scale b already reached.", "c": "Universities and nonprofits produced s1, and 16 H100 for 26 minutes is a few GPU-hours, within the academic 1 to 8 GPU budget given more wall time.", "d": "1,000 curated examples and a few GPU-hours on an open base fit a proof-of-concept budget, though the notes give no direct proof-of-concept anchor." }, "pipeline_role": "Consumes a's open bases and reasoning traces; c and b produce small specialised reasoning models that d turns into proofs of concept.", "t11_topic_number": 4, "sources": [ "https://arxiv.org/html/2410.23261v1", "https://arxiv.org/html/2501.19393v1", "https://arxiv.org/abs/2502.03387", "https://arxiv.org/abs/2505.03335" ], "unverified": [], "papers": [ { "title": "LIMA: Less Is More for Alignment", "authors": "Chunting Zhou et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.11206", "role": "founding" }, { "title": "STaR: Bootstrapping Reasoning With Reasoning", "authors": "Eric Zelikman et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2203.14465", "role": "founding" }, { "title": "s1: Simple test-time scaling", "authors": "Niklas Muennighoff et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2501.19393", "role": "state of the art" }, { "title": "LIMO: Less is More for Reasoning", "authors": "Yixin Ye et al.", "year": 2025, "venue": "COLM 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2502.03387", "role": "state of the art" }, { "title": "Distilling Step-by-Step! Outperforming Larger Language Models with Less Training Data and Smaller Model Sizes", "authors": "Cheng-Yu Hsieh et al.", "year": 2023, "venue": "ACL 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2305.02301", "role": "state of the art" } ], "approved": true }, { "id": "small-language-models", "name": "Small language models (SLM)", "definition": "Models under about 10B parameters that fit consumer devices.", "compute_anchor": "SmolLM3 3B: 384 H100 for 24 days from scratch; fine-tuning an SLM needs a few GPU-hours; serving a 7B model is 10 to 30x cheaper than 70B to 175B.", "data_anchor": "Phi-3-mini 3.3T tokens; SmolLM3 about 11T tokens.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): NVIDIA position paper has 8 authors.", "time_horizon": "Months.", "shelf_life_signals": [ "40 to 70% of LLM calls in three open agent systems could be replaced by SLMs (NVIDIA case-study estimate)" ], "exemplars": [ "Phi-3", "SmolLM3", "Qwen3 small", "gpt-oss-20b" ], "observed_actors": [ "microsoft", "hugging-face", "alibaba", "openai", "nvidia" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Microsoft, Hugging Face, Alibaba, OpenAI and NVIDIA are observed producing SLMs.", "b": "Training from scratch needs about 384 H100 for 24 days (SmolLM3), below the 70B scale b reached with Sahabat-AI, though the notes verify no Indonesian national compute programme, BRIN facility or ITB GPU inventory (T11 Part E item 2).", "c": "From-scratch training at 384 H100 for 24 days exceeds the academic 1 to 8 GPUs, while fine-tuning at a few GPU-hours fits.", "d": "Fine-tuning needs a few GPU-hours and serving is 10 to 30x cheaper than large models, which suits proof-of-concept delivery (not from-scratch training)." }, "pipeline_role": "a and open labs release SLMs; b and c fine-tune them for local languages and tasks; d deploys them on devices for proof of value.", "t11_topic_number": 6, "sources": [ "https://arxiv.org/abs/2505.09388", "https://arxiv.org/abs/2404.14219", "https://huggingface.co/blog/smollm3", "https://arxiv.org/html/2506.02153v1", "https://openai.com/index/introducing-gpt-oss/" ], "unverified": [], "papers": [ { "title": "SmolLM2: When Smol Goes Big -- Data-Centric Training of a Small Language Model", "authors": "Loubna Ben Allal et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2502.02737", "role": "state of the art" }, { "title": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone", "authors": "Marah Abdin et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2404.14219", "role": "state of the art" }, { "title": "Gemma 2: Improving Open Language Models at a Practical Size", "authors": "Gemma Team et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2408.00118", "role": "state of the art" }, { "title": "MobileLLM: Optimizing Sub-billion Parameter Language Models for On-Device Use Cases", "authors": "Zechun Liu et al.", "year": 2024, "venue": "ICML 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2402.14905", "role": "state of the art" }, { "title": "A Survey of Small Language Models", "authors": "Chien Van Nguyen et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2410.20011", "role": "survey" } ], "approved": true }, { "id": "tabular-foundation-models", "name": "Tabular foundation models", "definition": "In-context learning on tables.", "compute_anchor": "TabPFN: pretrained about 2 weeks on 8 GPUs; inference 2.8 s vs a 4-hour tuned ensemble.", "data_anchor": "About 130M synthetic datasets from structural causal models; handles up to 10,000 samples and 500 features.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Months.", "shelf_life_signals": [], "exemplars": [ "TabPFN" ], "observed_actors": [ "ellis-tubingen", "Academic group" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Pretraining needs about 8 GPUs for 2 weeks, trivial at frontier scale.", "b": "8 GPUs for 2 weeks on synthetic data is modest.", "c": "An academic group (ELLIS Tübingen) produced TabPFN on 8 GPUs, matching the academic 1 to 8 GPU range.", "d": "TabPFN is fine-tunable and answers in 2.8 s on up to 10,000 samples, suiting proof-of-value work on client tables." }, "pipeline_role": "c and academic groups produce tabular models; b and d apply and fine-tune them on sector tables.", "t11_topic_number": 22, "sources": [ "https://arxiv.org/html/2410.23261v1", "https://www.nature.com/articles/s41586-024-08328-6" ], "unverified": [], "papers": [ { "title": "TabPFN: A Transformer That Solves Small Tabular Classification Problems in a Second", "authors": "Noah Hollmann et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2207.01848", "role": "founding" }, { "title": "TabICL: A Tabular Foundation Model for In-Context Learning on Large Data", "authors": "Jingang Qu et al.", "year": 2025, "venue": "ICML 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2502.05564", "role": "state of the art" }, { "title": "TabArena: A Living Benchmark for Machine Learning on Tabular Data", "authors": "Nick Erickson et al.", "year": 2025, "venue": "NeurIPS 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2506.16791", "role": "state of the art" }, { "title": "Why do tree-based models still outperform deep learning on tabular data?", "authors": "Léo Grinsztajn et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2207.08815", "role": "state of the art" }, { "title": "Tabular Data: Deep Learning is Not All You Need", "authors": "Ravid Shwartz-Ziv et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2106.03253", "role": "survey" } ], "approved": true }, { "id": "time-series-foundation-models", "name": "Time-series foundation models", "definition": "Pretrained forecasters used zero-shot or fine-tuned.", "compute_anchor": "Chronos: T5-based, 20M to 710M parameters; training compute not given.", "data_anchor": "Public plus synthetic Gaussian-process data; 42-dataset benchmark.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Months.", "shelf_life_signals": [], "exemplars": [ "Chronos" ], "observed_actors": [ "amazon" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Amazon is the developer.", "b": "Models of 20M to 710M parameters are small, though training compute is not given in the notes.", "c": "Models of 20M to 710M parameters are small, though training compute is not given in the notes.", "d": "Zero-shot use of a pretrained forecaster on unseen data fits proof-of-concept work; training compute is not given." }, "pipeline_role": "a and big tech release pretrained forecasters; c benchmarks and fine-tunes them; d applies them zero-shot in proofs of value.", "t11_topic_number": 23, "sources": [ "https://arxiv.org/abs/2403.07815" ], "unverified": [], "papers": [ { "title": "A decoder-only foundation model for time-series forecasting", "authors": "Abhimanyu Das et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2310.10688", "role": "founding" }, { "title": "Chronos: Learning the Language of Time Series", "authors": "Abdul Fatir Ansari et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.07815", "role": "state of the art" }, { "title": "Unified Training of Universal Time Series Forecasting Transformers", "authors": "Gerald Woo et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.02592", "role": "state of the art" }, { "title": "Lag-Llama: Towards Foundation Models for Probabilistic Time Series Forecasting", "authors": "Kashif Rasul et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2310.08278", "role": "state of the art" }, { "title": "Foundation Models for Time Series Analysis: A Tutorial and Survey", "authors": "Yuxuan Liang et al.", "year": 2024, "venue": "KDD'24 (per arXiv comment)", "url": "https://arxiv.org/abs/2403.14735", "role": "survey" } ], "approved": true }, { "id": "world-models-interactive-generation", "name": "World models (generative interactive environments)", "definition": "Models that simulate environments for agents or users.", "compute_anchor": "Compute not published.", "data_anchor": "Not published.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Multi-year.", "shelf_life_signals": [], "exemplars": [ "Genie 3", "Marble" ], "observed_actors": [ "google-deepmind", "world-labs" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "not feasible", "tier_d": "not feasible", "tier_rationale": { "a": "Google DeepMind (Genie 3) and World Labs (Marble) are observed.", "b": "The notes publish no compute or data figure and show only frontier or funded actors over a multi-year horizon; the notes verify no Indonesian national compute programme, BRIN facility or ITB GPU inventory (T11 Part E item 2).", "c": "The notes publish no compute or data figure; graded on the multi-year horizon and frontier-only actors.", "d": "The notes publish no compute figure for building these; d could only use commercial products such as Marble (free to $95 per month)." }, "pipeline_role": "a produces world models; d may use commercial ones such as Marble; the notes show no b or c role.", "t11_topic_number": 18, "sources": [ "https://techcrunch.com/2025/08/05/deepmind-reveals-genie-3-a-world-model-that-could-be-the-key-to-reaching-agi", "https://siliconangle.com/2025/11/12/world-labs-launches-marble-commercial-world-model-generating-entire-virtual-environments/" ], "unverified": [], "papers": [ { "title": "World Models", "authors": "David Ha et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1803.10122", "role": "founding" }, { "title": "Genie: Generative Interactive Environments", "authors": "Jake Bruce et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.15391", "role": "founding" }, { "title": "Diffusion Models Are Real-Time Game Engines", "authors": "Dani Valevski et al.", "year": 2024, "venue": "ICLR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2408.14837", "role": "state of the art" }, { "title": "Understanding World or Predicting Future? A Comprehensive Survey of World Models", "authors": "Jingtao Ding et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2411.14499", "role": "survey" } ], "approved": true }, { "id": "world-models-self-supervised-video-robotics", "name": "World models (self-supervised video, robotics)", "definition": "Predictive representation models for planning.", "compute_anchor": "V-JEPA 2: 1.2B parameters; Cosmos sizes and compute not stated.", "data_anchor": "V-JEPA 2: over 1M hours of video plus 1M images, then 62 hours of robot data.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not sourced in the notes.", "time_horizon": "Multi-year.", "shelf_life_signals": [], "exemplars": [ "V-JEPA 2", "NVIDIA Cosmos" ], "observed_actors": [ "meta", "nvidia" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "stretch", "tier_rationale": { "a": "Meta and NVIDIA are observed and released open weights.", "b": "Pretraining on 1M hours of video is out of reach on the evidence, but the action-conditioned stage needs only 62 hours of robot data on open weights.", "c": "The 62-hour robot-data stage on an open 1.2B model is small, but robotics hardware and compute are not given in the notes.", "d": "The low data need on open weights suggests a possible proof of technology, but the notes give no compute or hardware anchor." }, "pipeline_role": "a releases open world models; b and c adapt them with small robot datasets; d could carry them into robotics proofs of technology.", "t11_topic_number": 19, "sources": [ "https://ai.meta.com/blog/v-jepa-2-world-model-benchmarks/", "https://arxiv.org/abs/2501.03575" ], "unverified": [], "papers": [ { "title": "Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture", "authors": "Mahmoud Assran et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2301.08243", "role": "founding" }, { "title": "V-JEPA 2: Self-Supervised Video Models Enable Understanding, Prediction and Planning", "authors": "Mido Assran et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2506.09985", "role": "state of the art" }, { "title": "Revisiting Feature Prediction for Learning Visual Representations from Video", "authors": "Adrien Bardes et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2404.08471", "role": "state of the art" }, { "title": "Mastering Diverse Domains through World Models", "authors": "Danijar Hafner et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2301.04104", "role": "state of the art" } ], "approved": true }, { "id": "real-time-object-detection", "name": "Real-time object detection (YOLO and DETR lines)", "definition": "Detectors that find and box objects in images fast enough for live camera feeds on one GPU or an edge device.", "compute_anchor": "RT-DETR-R50 and R101 run at 108 and 74 FPS on a T4 GPU; training compute is not stated in the abstract.", "data_anchor": "COCO, with Objects365 pretraining lifting RT-DETR-R50 and R101 to 55.3% and 56.2% AP.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 8 authors on the RT-DETR paper (arXiv 2304.08069).", "time_horizon": "Months: new YOLO and DETR variants appear several times a year.", "shelf_life_signals": [ "A new YOLO or DETR release overtakes the speed and accuracy curve on COCO", "Open-vocabulary detectors reach the same speed and absorb closed-set training" ], "exemplars": [ "baidu-rt-detr", "roboflow-rf-detr", "ultralytics-yolo26", "tsinghua-university-yolov10", "ustc-d-fine" ], "observed_actors": [ "baidu", "peking-university", "roboflow", "ultralytics", "tsinghua-university", "university-of-science-and-technology-of-china", "hefei-institute-of-artificial-intelligence" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs and big tech release the reference detectors, for example RT-DETR at 108 FPS on a T4.", "b": "Fine-tuning a detector that runs at 74 to 108 FPS on a T4 GPU on local data is within reach of an Indonesian lab.", "c": "A university unit can train and benchmark these detectors on COCO-scale data with a few GPUs, as the 8-author RT-DETR paper shows.", "d": "Pretrained real-time detectors fine-tuned on client images fit a proof of concept, with T4-class inference stated in the anchor." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and big tech release real-time detectors; Tier C benchmarks and adapts them to local datasets; Tier D fine-tunes them for client proofs of value such as counting and inspection; Tier B packages local datasets.", "sources": [ "https://arxiv.org/abs/2304.08069" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "You Only Look Once: Unified, Real-Time Object Detection", "authors": "Joseph Redmon et al.", "year": 2015, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1506.02640", "role": "founding" }, { "title": "YOLOv10: Real-Time End-to-End Object Detection", "authors": "Ao Wang et al.", "year": 2024, "venue": "NeurIPS 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2405.14458", "role": "state of the art" }, { "title": "DETRs Beat YOLOs on Real-time Object Detection", "authors": "Yian Zhao et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2304.08069", "role": "state of the art" }, { "title": "End-to-End Object Detection with Transformers", "authors": "Nicolas Carion et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2005.12872", "role": "state of the art" }, { "title": "A Comprehensive Review of YOLO Architectures in Computer Vision: From YOLOv1 to YOLOv8 and YOLO-NAS", "authors": "Juan Terven et al.", "year": 2023, "venue": "Mach. Learn. Knowl. Extr. 2023, 5, 1680-1716", "url": "https://arxiv.org/abs/2304.00501", "role": "survey" } ], "approved": true }, { "id": "open-vocabulary-detection-and-grounding", "name": "Open-vocabulary object detection and grounding", "definition": "Detectors and segmenters that find objects named in free text, including classes never labelled in training.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Grounding DINO reaches 52.5 AP zero-shot on COCO without COCO training data and 26.1 mean AP on ODinW.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 12 authors on the Grounding DINO paper (arXiv 2303.05499).", "time_horizon": "Months to a year.", "shelf_life_signals": [ "Multimodal LLMs that output boxes directly match dedicated open-set detectors", "Real-time open-vocabulary detectors reach closed-set accuracy" ], "exemplars": [ "idea-research-grounding-dino", "tencent-yolo-world", "google-owlv2-large-patch14", "tsinghua-university-yoloe" ], "observed_actors": [ "idea-research", "tencent", "google", "tsinghua-university" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Grounded pretraining on large image-text data, as in Grounding DINO, is frontier and big-tech work.", "b": "Pretraining a grounding model is a stretch for an Indonesian lab; adapting a released one with local labels is feasible.", "c": "Evaluating zero-shot transfer on COCO and ODinW, as the 12-author Grounding DINO paper does, fits a university unit.", "d": "Zero-shot detection at 52.5 COCO AP lets a delivery team show a detector on client classes without labelling first." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A pretrains grounded detectors; Tier C tests zero-shot transfer on local classes; Tier D uses them to label client images and to demo detection without training data; Tier B adds Indonesian object vocabularies.", "sources": [ "https://arxiv.org/abs/2303.05499" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Grounded Language-Image Pre-training", "authors": "Liunian Harold Li et al.", "year": 2021, "venue": "CVPR 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2112.03857", "role": "founding" }, { "title": "YOLO-World: Real-Time Open-Vocabulary Object Detection", "authors": "Tianheng Cheng et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2401.17270", "role": "state of the art" }, { "title": "Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection", "authors": "Shilong Liu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2303.05499", "role": "state of the art" }, { "title": "Simple Open-Vocabulary Object Detection with Vision Transformers", "authors": "Matthias Minderer et al.", "year": 2022, "venue": "ECCV 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2205.06230", "role": "state of the art" }, { "title": "A Survey on Open-Vocabulary Detection and Segmentation: Past, Present, and Future", "authors": "Chaoyang Zhu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2307.09220", "role": "survey" } ], "approved": true }, { "id": "promptable-segmentation-foundation-models", "name": "Promptable segmentation foundation models (Segment Anything)", "definition": "Models that cut out any object from a point, box or text prompt, trained on very large mask collections.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "SA-1B: over 1 billion masks on 11M licensed and privacy-respecting images.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 12 authors on the Segment Anything paper (arXiv 2304.02643).", "time_horizon": "Year.", "shelf_life_signals": [ "A new SAM generation adds video or concept prompts", "Distilled SAM variants reach on-device speed" ], "exemplars": [ "meta-sam-1", "meta-sam-2", "meta-sam-3", "mobilesam", "bo-wang-lab-medsam" ], "observed_actors": [ "meta", "kyung-hee-university", "bo-wang-lab" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Building a 1-billion-mask data engine over 11M images is frontier-lab scale.", "b": "Repeating SA-1B scale is beyond an Indonesian lab; fine-tuning SAM on local imagery is feasible.", "c": "Domain adaptation of SAM, as MedSAM did, fits a university unit.", "d": "Prompted segmentation from a released SAM speeds up labelling and measurement in proofs of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A releases SAM models and SA-1B; Tier C adapts them to domains such as medicine; Tier D uses them for labelling and measurement in client pilots.", "sources": [ "https://arxiv.org/abs/2304.02643" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Segment Anything", "authors": "Alexander Kirillov et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2304.02643", "role": "founding" }, { "title": "SAM 3: Segment Anything with Concepts", "authors": "Nicolas Carion et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2511.16719", "role": "state of the art" }, { "title": "SAM 2: Segment Anything in Images and Videos", "authors": "Nikhila Ravi et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2408.00714", "role": "state of the art" }, { "title": "Faster Segment Anything: Towards Lightweight SAM for Mobile Applications", "authors": "Chaoning Zhang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.14289", "role": "state of the art" }, { "title": "EfficientSAM: Leveraged Masked Image Pretraining for Efficient Segment Anything", "authors": "Yunyang Xiong et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.00863", "role": "state of the art" } ], "approved": true }, { "id": "universal-image-segmentation", "name": "Semantic, instance and panoptic segmentation architectures", "definition": "Networks that label every pixel of an image by class, by object instance, or both.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Mask2Former reports 57.8 PQ on COCO panoptic, 50.1 AP on COCO instance and 57.7 mIoU on ADE20K.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on the Mask2Former paper (arXiv 2112.01527).", "time_horizon": "Year.", "shelf_life_signals": [ "Promptable and open-vocabulary segmenters replace closed-set models", "One model handles all three segmentation tasks" ], "exemplars": [ "meta-mask2former", "segformer", "google-deeplabv3-plus", "mask-r-cnn" ], "observed_actors": [ "meta", "nvidia", "google" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Big-tech labs set the architecture line, from Mask R-CNN to Mask2Former.", "b": "Training these networks on COCO and ADE20K is within reach of an Indonesian lab.", "c": "A 5-author team produced Mask2Former, a size a university unit can match.", "d": "Fine-tuning a released segmenter on client imagery fits a proof of concept." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A sets architectures; Tier C trains them on local datasets; Tier D applies them to area, defect and land-cover measurement in pilots.", "sources": [ "https://arxiv.org/abs/2112.01527" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Mask R-CNN", "authors": "Kaiming He et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1703.06870", "role": "founding" }, { "title": "Panoptic Segmentation", "authors": "Alexander Kirillov et al.", "year": 2018, "venue": "CVPR 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1801.00868", "role": "founding" }, { "title": "OneFormer: One Transformer to Rule Universal Image Segmentation", "authors": "Jitesh Jain et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2211.06220", "role": "state of the art" }, { "title": "Per-Pixel Classification is Not All You Need for Semantic Segmentation", "authors": "Bowen Cheng et al.", "year": 2021, "venue": "NeurIPS 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2107.06278", "role": "state of the art" }, { "title": "Masked-attention Mask Transformer for Universal Image Segmentation", "authors": "Bowen Cheng et al.", "year": 2021, "venue": "CVPR 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2112.01527", "role": "state of the art" } ], "approved": true }, { "id": "self-supervised-visual-representation-learning", "name": "Self-supervised visual representation learning", "definition": "Pretraining image encoders without labels so their features transfer to many vision tasks.", "compute_anchor": "DINOv3 scales both dataset and model size; the 7B teacher is named in the Hugging Face checkpoint dinov3-vit7b16-pretrain-lvd1689m. Training compute is not in the sources read.", "data_anchor": "The checkpoint name refers to the LVD-1689M dataset; the abstract states data from natural to aerial images.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 26 authors on the DINOv3 report (arXiv 2508.10104).", "time_horizon": "Year.", "shelf_life_signals": [ "A new DINO or JEPA generation becomes the default backbone", "Vision-language encoders overtake label-free ones on dense tasks" ], "exemplars": [ "meta-dinov3", "meta-dinov2", "mae", "i-jepa" ], "observed_actors": [ "meta" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "A 7B teacher on a dataset named LVD-1689M, built by a 26-author team, is frontier-lab work.", "b": "Pretraining at that scale is not feasible for an Indonesian lab; using the released distilled backbones is.", "c": "Small-scale self-supervised studies on local data fit a university unit; frontier scale does not.", "d": "Released DINOv3 backbones give strong features for few-label client tasks." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A pretrains and releases backbones; Tier C studies them on local imagery such as aerial data; Tier D uses frozen features to build classifiers with few labels.", "sources": [ "https://arxiv.org/abs/2508.10104", "https://huggingface.co/facebook/dinov3-vit7b16-pretrain-lvd1689m" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "A Simple Framework for Contrastive Learning of Visual Representations", "authors": "Ting Chen et al.", "year": 2020, "venue": "ICML'2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2002.05709", "role": "founding" }, { "title": "Momentum Contrast for Unsupervised Visual Representation Learning", "authors": "Kaiming He et al.", "year": 2019, "venue": "CVPR 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/1911.05722", "role": "founding" }, { "title": "DINOv3", "authors": "Oriane Siméoni et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2508.10104", "role": "state of the art" }, { "title": "DINOv2: Learning Robust Visual Features without Supervision", "authors": "Maxime Oquab et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2304.07193", "role": "state of the art" }, { "title": "Emerging Properties in Self-Supervised Vision Transformers", "authors": "Mathilde Caron et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2104.14294", "role": "state of the art" } ], "approved": true }, { "id": "contrastive-image-text-encoders", "name": "Contrastive image-text encoders (CLIP and SigLIP lines)", "definition": "Image and text encoders trained together so that pictures and captions share one embedding space for zero-shot search and classification.", "compute_anchor": "A SigLiT model reaches 84.5% ImageNet zero-shot accuracy with four TPUv4 chips in two days.", "data_anchor": "Batch sizes up to one million were tested; 32k was found sufficient.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on the SigLIP paper (arXiv 2303.15343).", "time_horizon": "Year.", "shelf_life_signals": [ "Encoders are absorbed into multimodal LLM vision towers", "Domain encoders (fashion, medicine) outperform general ones" ], "exemplars": [ "siglip-400m", "align", "google-tipsv2", "fashion-clip" ], "observed_actors": [ "google" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Web-scale image-text pretraining is led by big tech.", "b": "Four TPUv4 chips for two days, as in the SigLIP anchor, is within an Indonesian lab budget for locked-image tuning.", "c": "A 4-author paper shows a university unit can run this research.", "d": "Zero-shot image search and tagging from a released encoder fits a proof of concept, for example product search." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A trains web-scale encoders; Tier B and Tier C tune them on Indonesian captions and domains; Tier D uses them for search and tagging pilots.", "sources": [ "https://arxiv.org/abs/2303.15343" ], "unverified": [], "papers": [ { "title": "Learning Transferable Visual Models From Natural Language Supervision", "authors": "Alec Radford et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2103.00020", "role": "founding" }, { "title": "Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision", "authors": "Chao Jia et al.", "year": 2021, "venue": "International Conference on Machine Learning 2021", "url": "https://arxiv.org/abs/2102.05918", "role": "founding" }, { "title": "SigLIP 2: Multilingual Vision-Language Encoders with Improved Semantic Understanding, Localization, and Dense Features", "authors": "Michael Tschannen et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2502.14786", "role": "state of the art" }, { "title": "Sigmoid Loss for Language Image Pre-Training", "authors": "Xiaohua Zhai et al.", "year": 2023, "venue": "ICCV'23 (per arXiv comment)", "url": "https://arxiv.org/abs/2303.15343", "role": "state of the art" }, { "title": "Reproducible scaling laws for contrastive language-image learning", "authors": "Mehdi Cherti et al.", "year": 2022, "venue": "Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognitio", "url": "https://arxiv.org/abs/2212.07143", "role": "state of the art" } ], "approved": true }, { "id": "vision-backbone-architectures", "name": "Image classification backbone architectures (CNN and vision transformer)", "definition": "Designing the core networks that turn an image into features, from residual CNNs to vision transformers.", "compute_anchor": "ViT needs substantially fewer computational resources to train than comparable CNNs when pretrained on large data, per the abstract; no figure is given there.", "data_anchor": "Pretraining on large data, then transfer to ImageNet, CIFAR-100 and VTAB.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 12 authors on the ViT paper (arXiv 2010.11929).", "time_horizon": "Years: classic backbones stay in use long after newer ones appear.", "shelf_life_signals": [ "A new architecture family (for example state-space or large-kernel) tops ImageNet", "Backbone choice moves to pretrained foundation encoders" ], "exemplars": [ "google-vit", "meta-convnext", "microsoft-swin-transformer", "resnet50" ], "observed_actors": [ "google", "meta", "microsoft" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Big-tech labs introduced ViT and Swin with large pretraining sets.", "b": "Training and comparing backbones on ImageNet-scale data is within an Indonesian lab reach.", "c": "Many backbone papers come from small university teams; transfer to CIFAR-100 and VTAB fits a unit.", "d": "Pretrained backbones are the default starting point of any vision proof of concept." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and academia publish backbones; Tier C benchmarks them on local data; Tier D picks pretrained ones as the base of client models.", "sources": [ "https://arxiv.org/abs/2010.11929" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Deep Residual Learning for Image Recognition", "authors": "Kaiming He et al.", "year": 2015, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1512.03385", "role": "founding" }, { "title": "An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale", "authors": "Alexey Dosovitskiy et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2010.11929", "role": "founding" }, { "title": "A ConvNet for the 2020s", "authors": "Zhuang Liu et al.", "year": 2022, "venue": "CVPR 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2201.03545", "role": "state of the art" }, { "title": "Swin Transformer: Hierarchical Vision Transformer using Shifted Windows", "authors": "Ze Liu et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2103.14030", "role": "state of the art" } ], "approved": true }, { "id": "efficient-vision-networks-for-edge", "name": "Efficient vision networks for mobile and edge devices", "definition": "Small, fast vision networks, often found by architecture search, that run on phones, cameras and microcontrollers.", "compute_anchor": "EfficientNet-B7 is 8.4x smaller and 6.1x faster at inference than the best earlier ConvNet; training compute is not stated in the abstract.", "data_anchor": "ImageNet (84.3% top-1 for B7), with transfer to CIFAR-100 (91.7%) and Flowers (98.8%).", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 2 authors on the EfficientNet paper (arXiv 1905.11946).", "time_horizon": "Months to a year.", "shelf_life_signals": [ "New edge accelerators favour different operators", "Quantisation-aware designs beat larger float models on device" ], "exemplars": [ "google-efficientnet", "google-mobilenet-v4", "shufflenet-v2", "mobileone" ], "observed_actors": [ "google", "megvii", "apple" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Architecture search at Google scale produced EfficientNet and MobileNet.", "b": "Training small networks on ImageNet is feasible for an Indonesian lab.", "c": "A 2-author paper set this line, so a university unit can contribute.", "d": "Small models that run on site cameras and phones suit plantation and factory pilots with limited connectivity." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A searches and releases efficient networks; Tier C ports them to local hardware; Tier D deploys them on edge devices in pilots.", "sources": [ "https://arxiv.org/abs/1905.11946" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "MobileNetV2: Inverted Residuals and Linear Bottlenecks", "authors": "Mark Sandler et al.", "year": 2018, "venue": "The IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2018, pp.", "url": "https://arxiv.org/abs/1801.04381", "role": "founding" }, { "title": "EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks", "authors": "Mingxing Tan et al.", "year": 2019, "venue": "International Conference on Machine Learning, 2019", "url": "https://arxiv.org/abs/1905.11946", "role": "founding" }, { "title": "MobileNetV4 -- Universal Models for the Mobile Ecosystem", "authors": "Danfeng Qin et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2404.10518", "role": "state of the art" }, { "title": "EfficientViT: Multi-Scale Linear Attention for High-Resolution Dense Prediction", "authors": "Han Cai et al.", "year": 2022, "venue": "ICCV 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2205.14756", "role": "state of the art" }, { "title": "Searching for MobileNetV3", "authors": "Andrew Howard et al.", "year": 2019, "venue": "ICCV 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1905.02244", "role": "state of the art" } ], "approved": true }, { "id": "monocular-and-stereo-depth-estimation", "name": "Monocular and stereo depth estimation", "definition": "Estimating the distance of every pixel from one image, a video or a stereo pair.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Depth Anything scales training to about 62M automatically annotated unlabelled images, then fine-tunes on NYUv2 and KITTI for metric depth.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on the Depth Anything paper (arXiv 2401.10891).", "time_horizon": "Months to a year.", "shelf_life_signals": [ "Feed-forward 3D models predict depth and camera pose together", "Metric depth works zero-shot on new scenes" ], "exemplars": [ "depth-anything", "apple-depth-pro", "intel-zoedepth", "nvidia-foundationstereo" ], "observed_actors": [ "hku", "bytedance", "apple", "intel", "nvidia" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Data engines over about 62M images are big-tech scale.", "b": "Fine-tuning released depth models on local scenes is feasible; the data engine is a stretch for an Indonesian lab.", "c": "Fine-tuning on NYUv2 and KITTI, as in the anchor, fits a university unit.", "d": "Zero-shot depth from one camera helps measurement and volume estimation in proofs of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A releases depth foundation models; Tier C fine-tunes for local metric depth; Tier D uses them for stockpile, tree height and distance estimates in pilots.", "sources": [ "https://arxiv.org/abs/2401.10891" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer", "authors": "René Ranftl et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1907.01341", "role": "founding" }, { "title": "Depth Anything: Unleashing the Power of Large-Scale Unlabeled Data", "authors": "Lihe Yang et al.", "year": 2024, "venue": "CVPR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2401.10891", "role": "state of the art" }, { "title": "Depth Anything V2", "authors": "Lihe Yang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.09414", "role": "state of the art" }, { "title": "Metric3Dv2: A Versatile Monocular Geometric Foundation Model for Zero-shot Metric Depth and Surface Normal Estimation", "authors": "Mu Hu et al.", "year": 2024, "venue": "IEEE Transactions on Pattern Analysis and Machine Intelligence, vol. 46, no. 12,", "url": "https://arxiv.org/abs/2404.15506", "role": "state of the art" }, { "title": "Repurposing Diffusion-Based Image Generators for Monocular Depth Estimation", "authors": "Bingxin Ke et al.", "year": 2023, "venue": "CVPR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2312.02145", "role": "state of the art" } ], "approved": true }, { "id": "feed-forward-3d-reconstruction", "name": "Feed-forward 3D reconstruction and visual geometry", "definition": "Recovering camera poses, point clouds, matches and motion from images in one network pass instead of slow optimisation.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; VGGT infers cameras, point maps, depth and tracks from one, a few or hundreds of views.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on the VGGT paper (arXiv 2503.11651).", "time_horizon": "Months to a year.", "shelf_life_signals": [ "One feed-forward model replaces structure-from-motion pipelines", "Metric reconstruction works from phone video" ], "exemplars": [ "meta-vggt-1b", "naver-dust3r", "naver-mast3r", "meta-mapanything" ], "observed_actors": [ "meta", "naver" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Meta and NAVER LABS lead feed-forward geometry models such as VGGT and DUSt3R.", "b": "Training such a model is a stretch for an Indonesian lab; applying released ones is feasible.", "c": "A 6-author team produced VGGT, a size a university unit can match for follow-up work.", "d": "Released models turn photos into 3D scenes quickly for site survey and inspection demos." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A releases geometry models; Tier C evaluates them on local scenes; Tier D uses them for 3D site capture in proofs of concept.", "sources": [ "https://arxiv.org/abs/2503.11651" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "DUSt3R: Geometric 3D Vision Made Easy", "authors": "Shuzhe Wang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.14132", "role": "founding" }, { "title": "VGGT: Visual Geometry Grounded Transformer", "authors": "Jianyuan Wang et al.", "year": 2025, "venue": "CVPR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2503.11651", "role": "state of the art" }, { "title": "Fast3R: Towards 3D Reconstruction of 1000+ Images in One Forward Pass", "authors": "Jianing Yang et al.", "year": 2025, "venue": "CVPR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2501.13928", "role": "state of the art" }, { "title": "Grounding Image Matching in 3D with MASt3R", "authors": "Vincent Leroy et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.09756", "role": "state of the art" }, { "title": "pixelSplat: 3D Gaussian Splats from Image Pairs for Scalable Generalizable 3D Reconstruction", "authors": "David Charatan et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.12337", "role": "state of the art" } ], "approved": true }, { "id": "human-pose-estimation", "name": "Human pose estimation", "definition": "Locating body, hand and face keypoints of people in images and video.", "compute_anchor": "ViTPose scales from 100M to 1B parameters; training compute is not stated in the abstract.", "data_anchor": "Not in the sources read.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on the ViTPose paper (arXiv 2204.12484).", "time_horizon": "Year.", "shelf_life_signals": [ "Human-centric foundation models (for example Sapiens) replace task models", "On-device pose reaches desktop accuracy" ], "exemplars": [ "vitpose", "openmmlab-rtmpose", "openpose", "google-blazepose" ], "observed_actors": [ "university-of-sydney", "jd-com", "openmmlab", "carnegie-mellon-university", "google" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Big tech releases human-centric foundation models.", "b": "Training pose models of 100M parameters is within an Indonesian lab reach.", "c": "A 4-author team produced ViTPose, a size a university unit can match.", "d": "Released pose models support worker safety and ergonomics proofs of concept." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and academia publish pose models; Tier C adapts them to local settings; Tier D uses them for safety and activity monitoring pilots.", "sources": [ "https://arxiv.org/abs/2204.12484" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "OpenPose: Realtime Multi-Person 2D Pose Estimation using Part Affinity Fields", "authors": "Zhe Cao et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1812.08008", "role": "founding" }, { "title": "Sapiens: Foundation for Human Vision Models", "authors": "Rawal Khirodkar et al.", "year": 2024, "venue": "ECCV 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2408.12569", "role": "state of the art" }, { "title": "RTMPose: Real-Time Multi-Person Pose Estimation based on MMPose", "authors": "Tao Jiang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2303.07399", "role": "state of the art" }, { "title": "ViTPose: Simple Vision Transformer Baselines for Human Pose Estimation", "authors": "Yufei Xu et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2204.12484", "role": "state of the art" }, { "title": "Deep High-Resolution Representation Learning for Human Pose Estimation", "authors": "Ke Sun et al.", "year": 2019, "venue": "CVPR2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1902.09212", "role": "state of the art" } ], "approved": true }, { "id": "3d-human-body-modelling", "name": "3D human body models, mesh recovery and motion generation", "definition": "Parametric 3D models of the body, hands and face, fitted to images or generated from text.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "SMPL-X is trained from thousands of 3D scans.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 7 authors on the SMPL-X paper (arXiv 1904.05866).", "time_horizon": "Years.", "shelf_life_signals": [ "Text-to-motion models drive avatars and robots", "Body models are learned end to end from video" ], "exemplars": [ "smpl-x", "hmr", "mano-hand-model", "tencent-hy-motion-1-0" ], "observed_actors": [ "mpi-is", "uc-berkeley", "tencent" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Body models from thousands of scans come from labs such as MPI-IS.", "b": "Collecting scan data is a stretch for an Indonesian lab; fitting released models is feasible.", "c": "Mesh recovery research fits a university unit, as the 7-author SMPL-X paper shows.", "d": "Client value is mainly in ergonomics and animation, which needs specialist work." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A and research institutes build body models; Tier C applies them to motion analysis; Tier D uses them only in specialist pilots.", "sources": [ "https://arxiv.org/abs/1904.05866" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Expressive Body Capture: 3D Hands, Face, and Body from a Single Image", "authors": "Georgios Pavlakos et al.", "year": 2019, "venue": "CVPR 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1904.05866", "role": "founding" }, { "title": "End-to-end Recovery of Human Shape and Pose", "authors": "Angjoo Kanazawa et al.", "year": 2017, "venue": "CVPR 2018 (per arXiv comment)", "url": "https://arxiv.org/abs/1712.06584", "role": "founding" }, { "title": "Humans in 4D: Reconstructing and Tracking Humans with Transformers", "authors": "Shubham Goel et al.", "year": 2023, "venue": "ICCV 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2305.20091", "role": "state of the art" }, { "title": "SMPLer-X: Scaling Up Expressive Human Pose and Shape Estimation", "authors": "Zhongang Cai et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2309.17448", "role": "state of the art" }, { "title": "WHAM: Reconstructing World-grounded Humans with Accurate 3D Motion", "authors": "Soyong Shin et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.07531", "role": "state of the art" } ], "approved": true }, { "id": "face-analysis-and-liveness", "name": "Face detection, recognition and liveness", "definition": "Finding faces, verifying identity, and checking attributes, gaze or spoofing for identity and safety use.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; ArcFace addresses massive label noise in raw web faces.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on the ArcFace paper (arXiv 1801.07698).", "time_horizon": "Years.", "shelf_life_signals": [ "Regulation tightens use of face recognition", "Deepfake attacks force new liveness checks" ], "exemplars": [ "insightface-arcface", "insightface-retinaface", "google-blazeface", "anti-spoof-mn3" ], "observed_actors": [ "insightface", "google" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Academia and industry labs set face recognition methods such as ArcFace.", "b": "Training face models on local faces is within an Indonesian lab reach, subject to consent and data law.", "c": "A 6-author paper set the method; a university unit can work on it.", "d": "Released face and liveness models support eKYC and access pilots, with privacy review." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and academia publish methods; Tier B and Tier C test them on Indonesian faces; Tier D builds eKYC and driver monitoring pilots.", "sources": [ "https://arxiv.org/abs/1801.07698" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "ArcFace: Additive Angular Margin Loss for Deep Face Recognition", "authors": "Jiankang Deng et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1801.07698", "role": "founding" }, { "title": "Joint Face Detection and Alignment using Multi-task Cascaded Convolutional Networks", "authors": "Kaipeng Zhang et al.", "year": 2016, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1604.02878", "role": "founding" }, { "title": "AdaFace: Quality Adaptive Margin for Face Recognition", "authors": "Minchul Kim et al.", "year": 2022, "venue": "CVPR2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2204.00964", "role": "state of the art" }, { "title": "RetinaFace: Single-stage Dense Face Localisation in the Wild", "authors": "Jiankang Deng et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1905.00641", "role": "state of the art" }, { "title": "Deep Learning for Face Anti-Spoofing: A Survey", "authors": "Zitong Yu et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2106.14948", "role": "survey" } ], "approved": true }, { "id": "video-action-recognition", "name": "Video action recognition and temporal understanding", "definition": "Classifying and locating human actions and events in video clips.", "compute_anchor": "VideoMAE V2 trains a video ViT with a billion parameters; training compute is not stated in the abstract.", "data_anchor": "Kinetics-400 (90.0%), Kinetics-600 (89.9%) and Something-Something V1 and V2 (68.7% and 77.0%).", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 8 authors on the VideoMAE V2 paper (arXiv 2303.16727).", "time_horizon": "Year.", "shelf_life_signals": [ "Video LLMs answer questions about events directly", "Video encoders become general foundation models" ], "exemplars": [ "videomae-v2", "meta-slowfast", "google-videoprism", "st-gcn" ], "observed_actors": [ "meta", "google", "cuhk" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Billion-parameter video pretraining is frontier and big-tech scale.", "b": "Pretraining at that scale is a stretch; fine-tuning released video models on local footage is feasible.", "c": "Benchmarking on Kinetics and Something-Something fits a university unit.", "d": "Released action models support safety and process monitoring pilots on client cameras." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A pretrains video models; Tier C fine-tunes them on local footage; Tier D applies them to unsafe-act and process detection pilots.", "sources": [ "https://arxiv.org/abs/2303.16727" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Two-Stream Convolutional Networks for Action Recognition in Videos", "authors": "Karen Simonyan et al.", "year": 2014, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1406.2199", "role": "founding" }, { "title": "Quo Vadis, Action Recognition? A New Model and the Kinetics Dataset", "authors": "Joao Carreira et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1705.07750", "role": "founding" }, { "title": "InternVideo2: Scaling Foundation Models for Multimodal Video Understanding", "authors": "Yi Wang et al.", "year": 2024, "venue": "ECCV2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2403.15377", "role": "state of the art" }, { "title": "VideoMAE: Masked Autoencoders are Data-Efficient Learners for Self-Supervised Video Pre-Training", "authors": "Zhan Tong et al.", "year": 2022, "venue": "NeurIPS 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2203.12602", "role": "state of the art" }, { "title": "Is Space-Time Attention All You Need for Video Understanding?", "authors": "Gedas Bertasius et al.", "year": 2021, "venue": "ICML 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2102.05095", "role": "state of the art" } ], "approved": true }, { "id": "multi-object-tracking-and-reidentification", "name": "Multi-object tracking and re-identification", "definition": "Following people, vehicles or animals across video frames and cameras and recognising them again.", "compute_anchor": "ByteTrack runs at 30 FPS on a single V100 GPU on MOT17.", "data_anchor": "MOT17, MOT20, HiEve and BDD100K tracking benchmarks.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 9 authors on the ByteTrack paper (arXiv 2110.06864).", "time_horizon": "Years.", "shelf_life_signals": [ "Trackers fold into detection models", "Privacy rules limit cross-camera re-identification of people" ], "exemplars": [ "bytetrack", "deepsort", "osnet", "oc-sort" ], "observed_actors": [ "hust", "university-of-surrey", "queen-mary-university-of-london", "carnegie-mellon-university", "shanghai-ai-lab", "nvidia" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Academic and industry labs publish trackers such as ByteTrack.", "b": "A tracker that runs at 30 FPS on one V100 is within an Indonesian lab reach.", "c": "Benchmarking on MOT17 and MOT20 fits a university unit.", "d": "Tracking turns detections into counts and flows for traffic, retail and plantation pilots." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and academia publish trackers; Tier C tests them on local footage; Tier D uses them for counting and flow analysis.", "sources": [ "https://arxiv.org/abs/2110.06864" ], "unverified": [], "papers": [ { "title": "Simple Online and Realtime Tracking", "authors": "Alex Bewley et al.", "year": 2016, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1602.00763", "role": "founding" }, { "title": "ByteTrack: Multi-Object Tracking by Associating Every Detection Box", "authors": "Yifu Zhang et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2110.06864", "role": "state of the art" }, { "title": "FairMOT: On the Fairness of Detection and Re-Identification in Multiple Object Tracking", "authors": "Yifu Zhang et al.", "year": 2020, "venue": "International Journal of Computer Vision, 129(11), 2021, 3069-3087", "url": "https://arxiv.org/abs/2004.01888", "role": "state of the art" }, { "title": "Simple Online and Realtime Tracking with a Deep Association Metric", "authors": "Nicolai Wojke et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1703.07402", "role": "state of the art" }, { "title": "Multiple Object Tracking: A Literature Review", "authors": "Wenhan Luo et al.", "year": 2014, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1409.7618", "role": "survey" } ], "approved": true }, { "id": "industrial-visual-anomaly-detection", "name": "Industrial visual anomaly and defect detection", "definition": "Finding defects on products and equipment by learning what normal looks like, often with few or no defect examples.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "MVTec AD benchmark, where PatchCore reaches up to 99.6% image-level AUROC.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on the PatchCore paper (arXiv 2106.08265).", "time_horizon": "Months to a year.", "shelf_life_signals": [ "Zero-shot anomaly detection with vision-language models", "MVTec AD saturates and new benchmarks replace it" ], "exemplars": [ "amazon-patchcore", "padim", "efficientad", "anomalib" ], "observed_actors": [ "amazon", "cea", "mvtec" ], "tier_a": "stretch", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "This is applied work; frontier labs rarely lead it, though Amazon published PatchCore.", "b": "Methods need only normal images and modest compute, which suits an Indonesian lab.", "c": "A 6-author team set the reference result of 99.6% AUROC on MVTec AD; a university unit can work on it.", "d": "Training on normal images only fits client factories where defects are rare." }, "tier_confidence": { "a": "medium", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier C and industry publish methods; Tier B and Tier C build local benchmarks; Tier D runs inspection pilots on client lines.", "sources": [ "https://arxiv.org/abs/2106.08265" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Uninformed Students: Student-Teacher Anomaly Detection with Discriminative Latent Embeddings", "authors": "Paul Bergmann et al.", "year": 2019, "venue": "CVPR 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/1911.02357", "role": "founding" }, { "title": "AnomalyGPT: Detecting Industrial Anomalies Using Large Vision-Language Models", "authors": "Zhaopeng Gu et al.", "year": 2023, "venue": "AAAI 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2308.15366", "role": "state of the art" }, { "title": "Towards Total Recall in Industrial Anomaly Detection", "authors": "Karsten Roth et al.", "year": 2021, "venue": "CVPR 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2106.08265", "role": "state of the art" }, { "title": "PaDiM: a Patch Distribution Modeling Framework for Anomaly Detection and Localization", "authors": "Thomas Defard et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2011.08785", "role": "state of the art" }, { "title": "Deep Industrial Image Anomaly Detection: A Survey", "authors": "Jiaqi Liu et al.", "year": 2023, "venue": "Machine Intelligence Research vol. 21, no. 1, pp. 104-135, 2024", "url": "https://arxiv.org/abs/2301.11514", "role": "survey" } ], "approved": true }, { "id": "computational-pathology-foundation-models", "name": "Computational pathology foundation models", "definition": "Self-supervised encoders trained on whole-slide tissue images for diagnosis and biomarker tasks.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "UNI is pretrained on over 100 million tissue patches from over 100,000 H&E whole-slide images across 20 tissue types.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 20 authors on the UNI paper (arXiv 2308.15474).", "time_horizon": "Year.", "shelf_life_signals": [ "Slide-level multimodal models replace patch encoders", "Regulatory approval of a foundation-model-based diagnostic" ], "exemplars": [ "mahmood-lab-uni", "paige-virchow", "microsoft-prov-gigapath", "mahmood-lab-conch" ], "observed_actors": [ "mahmood-lab", "paige", "microsoft", "providence" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "stretch", "tier_rationale": { "a": "Big hospital and tech partnerships hold the 100,000-slide archives used for UNI.", "b": "An Indonesian lab lacks slide archives of that size; using released encoders is possible.", "c": "A university hospital unit can fine-tune released encoders on local slides.", "d": "Pilots need hospital data access and clinical oversight, so delivery is a stretch." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A and hospital labs pretrain encoders; Tier C validates them on local slides; Tier D supports hospital pilots under clinical governance.", "sources": [ "https://arxiv.org/abs/2308.15474" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Data Efficient and Weakly Supervised Computational Pathology on Whole Slide Images", "authors": "Ming Y. Lu et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2004.09666", "role": "founding" }, { "title": "Towards a Visual-Language Foundation Model for Computational Pathology", "authors": "Ming Y. Lu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2307.12914", "role": "state of the art" }, { "title": "A General-Purpose Self-Supervised Model for Computational Pathology", "authors": "Richard J. Chen et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2308.15474", "role": "state of the art" }, { "title": "Virchow: A Million-Slide Digital Pathology Foundation Model", "authors": "Eugene Vorontsov et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2309.07778", "role": "state of the art" }, { "title": "A Survey of Pathology Foundation Model: Progress and Future Directions", "authors": "Conghao Xiong et al.", "year": 2025, "venue": "IJCAI 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2504.04045", "role": "survey" } ], "approved": true }, { "id": "medical-image-segmentation-and-screening", "name": "Medical image segmentation and screening AI (radiology)", "definition": "Models that outline organs and lesions or flag disease on X-ray, CT, MRI and other scans.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "MedSAM uses 1,570,263 image-mask pairs across 10 imaging modalities and over 30 cancer types.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on the MedSAM paper (arXiv 2304.12306).", "time_horizon": "Year.", "shelf_life_signals": [ "Promptable medical segmenters replace per-task models", "National TB and cancer screening programmes adopt AI readers" ], "exemplars": [ "bo-wang-lab-medsam", "dkfz-nnu-net-v2", "qure-ai-qxr", "ugm-tbscreen-ai" ], "observed_actors": [ "bo-wang-lab", "dkfz", "helmholtz-imaging", "qure-ai", "universitas-gadjah-mada", "university-of-melbourne" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Big tech and hospital labs release medical encoders and segmenters.", "b": "An Indonesian team built TB screening on local data (UGM), so local training is feasible.", "c": "A 6-author team built MedSAM from public data; a university unit can do similar work.", "d": "Clinical pilots need hospital partners and regulatory review, so delivery is a stretch." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and hospital labs release models; Tier C and Tier B validate them on Indonesian scans; Tier D supports screening pilots with clinical partners.", "sources": [ "https://arxiv.org/abs/2304.12306" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "U-Net: Convolutional Networks for Biomedical Image Segmentation", "authors": "Olaf Ronneberger et al.", "year": 2015, "venue": "MICCAI 2015 (per arXiv comment)", "url": "https://arxiv.org/abs/1505.04597", "role": "founding" }, { "title": "Segment Anything in Medical Images", "authors": "Jun Ma et al.", "year": 2023, "venue": "Nature Communications 15, 654 (2024)", "url": "https://arxiv.org/abs/2304.12306", "role": "state of the art" }, { "title": "TotalSegmentator: robust segmentation of 104 anatomical structures in CT images", "authors": "Jakob Wasserthal et al.", "year": 2022, "venue": "Radiol Artif Intell 2023;5(5):e230024", "url": "https://arxiv.org/abs/2208.05868", "role": "state of the art" }, { "title": "TransUNet: Transformers Make Strong Encoders for Medical Image Segmentation", "authors": "Jieneng Chen et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2102.04306", "role": "state of the art" }, { "title": "nnU-Net: Self-adapting Framework for U-Net-Based Medical Image Segmentation", "authors": "Fabian Isensee et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1809.10486", "role": "state of the art" } ], "approved": true }, { "id": "lidar-and-multi-sensor-3d-perception", "name": "LiDAR and multi-sensor 3D perception", "definition": "Detecting and segmenting objects in 3D from LiDAR point clouds, cameras or both, mainly for driving and robotics.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; CenterPoint detects and tracks 3D objects as points in a point cloud.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on the CenterPoint paper (arXiv 2006.11275).", "time_horizon": "Years.", "shelf_life_signals": [ "End-to-end driving models absorb separate 3D detectors", "Camera-only systems match LiDAR accuracy" ], "exemplars": [ "centerpoint", "pointpillars", "bevfusion", "pointnet" ], "observed_actors": [ "ut-austin", "nutonomy", "massachusetts-institute-of-technology", "stanford-university" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Driving companies and big labs lead 3D perception with large sensor fleets.", "b": "An Indonesian lab can fine-tune on public LiDAR data but lacks fleet data.", "c": "A 3-author team produced CenterPoint; a university unit can work on it.", "d": "Client use, such as mining haul trucks or plantation machinery, needs sensor data collection first." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A and driving firms train on fleet data; Tier C adapts models to local sensors; Tier D pilots 3D perception where clients have LiDAR.", "sources": [ "https://arxiv.org/abs/2006.11275" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "PointNet: Deep Learning on Point Sets for 3D Classification and Segmentation", "authors": "Charles R. Qi et al.", "year": 2016, "venue": "CVPR 2017 (per arXiv comment)", "url": "https://arxiv.org/abs/1612.00593", "role": "founding" }, { "title": "VoxelNet: End-to-End Learning for Point Cloud Based 3D Object Detection", "authors": "Yin Zhou et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1711.06396", "role": "founding" }, { "title": "BEVFusion: Multi-Task Multi-Sensor Fusion with Unified Bird's-Eye View Representation", "authors": "Zhijian Liu et al.", "year": 2022, "venue": "ICRA 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2205.13542", "role": "state of the art" }, { "title": "Center-based 3D Object Detection and Tracking", "authors": "Tianwei Yin et al.", "year": 2020, "venue": "Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognitio", "url": "https://arxiv.org/abs/2006.11275", "role": "state of the art" }, { "title": "PointPillars: Fast Encoders for Object Detection from Point Clouds", "authors": "Alex H. Lang et al.", "year": 2018, "venue": "CVPR 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1812.05784", "role": "state of the art" } ], "approved": true }, { "id": "image-and-video-restoration", "name": "Image and video restoration and super-resolution", "definition": "Removing blur, noise and compression damage and raising the resolution of photos and video.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; Real-ESRGAN trains on purely synthetic degradations.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on the Real-ESRGAN paper (arXiv 2107.10833).", "time_horizon": "Year.", "shelf_life_signals": [ "Diffusion-based restorers replace GAN-based ones", "One-step video restoration runs in real time" ], "exemplars": [ "real-esrgan", "swinir", "bytedance-seedvr2", "basicvsr-plus-plus" ], "observed_actors": [ "tencent", "eth-zurich", "bytedance", "ntu-singapore" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Big tech releases diffusion restorers such as SeedVR.", "b": "Training GAN restorers on synthetic degradations needs no labelled data and fits an Indonesian lab.", "c": "A 4-author team produced Real-ESRGAN; a university unit can work on it.", "d": "Upscaling and cleaning camera and drone images fits proofs of concept." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and academia release restorers; Tier C tunes them to local cameras; Tier D uses them to improve low-quality field imagery.", "sources": [ "https://arxiv.org/abs/2107.10833" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "Image Super-Resolution Using Deep Convolutional Networks", "authors": "Chao Dong et al.", "year": 2014, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1501.00092", "role": "founding" }, { "title": "SwinIR: Image Restoration Using Swin Transformer", "authors": "Jingyun Liang et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2108.10257", "role": "state of the art" }, { "title": "Real-ESRGAN: Training Real-World Blind Super-Resolution with Pure Synthetic Data", "authors": "Xintao Wang et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2107.10833", "role": "state of the art" }, { "title": "Restormer: Efficient Transformer for High-Resolution Image Restoration", "authors": "Syed Waqas Zamir et al.", "year": 2021, "venue": "CVPR 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2111.09881", "role": "state of the art" }, { "title": "ESRGAN: Enhanced Super-Resolution Generative Adversarial Networks", "authors": "Xintao Wang et al.", "year": 2018, "venue": "ECCV 2018 (per arXiv comment)", "url": "https://arxiv.org/abs/1809.00219", "role": "state of the art" } ], "approved": true }, { "id": "image-matting-and-background-removal", "name": "Image matting and background removal", "definition": "Separating a foreground subject from its background at hair-level detail for photos and video.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 7 authors on the BiRefNet paper (arXiv 2401.03407).", "time_horizon": "Year.", "shelf_life_signals": [ "Promptable segmenters reach matting quality", "Image generators remove backgrounds as an edit" ], "exemplars": [ "birefnet", "bria-rmbg", "u2-net", "robust-video-matting" ], "observed_actors": [ "nankai-university", "bria-ai", "university-of-alberta", "university-of-washington", "bytedance" ], "tier_a": "stretch", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Mostly academic and small-company work.", "b": "Training on public matting sets fits an Indonesian lab.", "c": "A 7-author team produced BiRefNet; a university unit can match that.", "d": "Background removal for product photos is a quick retail proof of value." }, "tier_confidence": { "a": "medium", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier C and small firms publish matting models; Tier D uses them in e-commerce and media pilots.", "sources": [ "https://arxiv.org/abs/2401.03407" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "Deep Image Matting", "authors": "Ning Xu et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1703.03872", "role": "founding" }, { "title": "Bilateral Reference for High-Resolution Dichotomous Image Segmentation", "authors": "Peng Zheng et al.", "year": 2024, "venue": "Bilateral Reference for High-Resolution Dichotomous Image Segmentation. CAAI Art", "url": "https://arxiv.org/abs/2401.03407", "role": "state of the art" }, { "title": "Robust High-Resolution Video Matting with Temporal Guidance", "authors": "Shanchuan Lin et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2108.11515", "role": "state of the art" }, { "title": "U$^2$-Net: Going Deeper with Nested U-Structure for Salient Object Detection", "authors": "Xuebin Qin et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2005.09007", "role": "state of the art" }, { "title": "MODNet: Real-Time Trimap-Free Portrait Matting via Objective Decomposition", "authors": "Zhanghan Ke et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2011.11961", "role": "state of the art" } ], "approved": true }, { "id": "plantation-crop-and-wildlife-vision", "name": "Plantation, crop and wildlife monitoring vision", "definition": "Vision models that grade oil palm fruit, spot crop disease and count or identify plants and animals in the field.", "compute_anchor": "A YOLOv8 oil palm FFB grader trained in about 3.1 hours (Gunadarma University).", "data_anchor": "6,592 oil palm FFB images from Roboflow across ripeness categories; precision 94.5%, recall 94.7%, mAP 98%.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on the Gunadarma FFB paper (Jurnal Ilmiah Teknologi dan Rekayasa 30(3), 2025).", "time_horizon": "Months.", "shelf_life_signals": [ "A plantation reports a production deployment of FFB grading", "Drone and satellite models replace ground photos" ], "exemplars": [ "gunadarma-university-yolov8-ffb-grader", "upn-veteran-jawa-timur-yolov11-ffb-grader", "multimedia-university-tf-resnet50-ganoderma", "google-speciesnet" ], "observed_actors": [ "gunadarma-university", "upn-veteran-jawa-timur", "multimedia-university", "fgv-rd", "google" ], "tier_a": "not feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs do not work on oil palm grading; Google publishes general wildlife and crop models.", "b": "Training took about 3.1 hours on 6,592 images, well within an Indonesian lab budget.", "c": "Indonesian and Malaysian universities already publish these models.", "d": "FFB grading and Ganoderma detection are direct plantation proofs of value; no plantation deployment is stated in the source." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier C builds datasets and models with plantations; Tier B scales them; Tier D runs grading and disease pilots at mills and estates.", "sources": [ "https://ejournal.gunadarma.ac.id/index.php/tekno/article/view/67" ], "unverified": [], "papers": [ { "title": "An open access repository of images on plant health to enable the development of mobile disease diagnostics", "authors": "David. P. Hughes et al.", "year": 2015, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1511.08060", "role": "founding" }, { "title": "BioCLIP: A Vision Foundation Model for the Tree of Life", "authors": "Samuel Stevens et al.", "year": 2023, "venue": "CVPR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2311.18803", "role": "state of the art" }, { "title": "Efficient Pipeline for Camera Trap Image Review", "authors": "Sara Beery et al.", "year": 2019, "venue": "KDD19 (per arXiv comment)", "url": "https://arxiv.org/abs/1907.06772", "role": "state of the art" }, { "title": "The iNaturalist Species Classification and Detection Dataset", "authors": "Grant Van Horn et al.", "year": 2017, "venue": "CVPR 2018 (per arXiv comment)", "url": "https://arxiv.org/abs/1707.06642", "role": "state of the art" }, { "title": "Deep learning in agriculture: A survey", "authors": "Andreas Kamilaris et al.", "year": 2018, "venue": "Computers and Electronics in Agriculture International Journal, 2018", "url": "https://arxiv.org/abs/1807.11809", "role": "survey" } ], "approved": true }, { "id": "edge-video-analytics", "name": "Pretrained video analytics for people, vehicles, retail and safety", "definition": "Ready-trained detectors and classifiers for cameras that count people, read plates, check safety gear and watch shelves.", "compute_anchor": "Intel person-detection-0200: 1.817M parameters and 0.786 GFLOPs at 256x256 input; training compute not given.", "data_anchor": "Internal test set (AP 0.2398 at IoU 0.50:0.95); training data size not given.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 2 authors on the CPPE-5 dataset paper (arXiv 2112.09569).", "time_horizon": "Months.", "shelf_life_signals": [ "Vendor model zoos are replaced by open-vocabulary detectors", "Privacy rules limit people analytics" ], "exemplars": [ "intel-person-detection", "nvidia-peoplenet", "nvidia-lpdnet-lprnet", "qualcomm-gearguardnet" ], "observed_actors": [ "intel", "nvidia", "qualcomm" ], "tier_a": "stretch", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Chip vendors such as Intel, NVIDIA and Qualcomm publish these models to sell hardware, not frontier labs.", "b": "Small models of about 1.8M parameters are easy to retrain on Indonesian footage.", "c": "A university unit can retrain them, and the 2-author CPPE-5 paper shows dataset work at that scale.", "d": "Ready-trained camera models give fast proofs of value in traffic, retail and safety." }, "tier_confidence": { "a": "medium", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Vendors release model zoos; Tier C retrains them on local footage; Tier D deploys them on client cameras.", "sources": [ "https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/models/intel/person-detection-0200/README.md", "https://arxiv.org/abs/2112.09569" ], "unverified": [], "papers": [ { "title": "NoScope: Optimizing Neural Network Queries over Video at Scale", "authors": "Daniel Kang et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1703.02529", "role": "founding" }, { "title": "MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications", "authors": "Andrew G. Howard et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1704.04861", "role": "founding" }, { "title": "Edge Intelligence: Paving the Last Mile of Artificial Intelligence with Edge Computing", "authors": "Zhi Zhou et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1905.10083", "role": "survey" } ], "approved": true }, { "id": "scene-text-detection-and-recognition", "name": "Scene and document text detection and recognition (OCR)", "definition": "Finding text regions in images and reading the characters, from street signs to scanned pages.", "compute_anchor": "PP-OCRv6 spans 1.5M to 34.5M parameters per its title; training compute is not stated in the abstract.", "data_anchor": "In-house benchmarks: PP-OCRv6_medium reaches 83.2% recognition accuracy and 86.2% detection Hmean.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 16 authors on the PP-OCRv6 paper (arXiv 2606.13108).", "time_horizon": "Months to a year.", "shelf_life_signals": [ "Document VLMs overtake pipeline OCR", "Small OCR models match billion-scale VLMs" ], "exemplars": [ "baidu-pp-ocrv6", "baidu-pp-ocr", "dbnet", "easyocr", "microsoft-trocr" ], "observed_actors": [ "baidu", "hust", "jaided-ai", "microsoft" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Baidu and big labs lead OCR systems.", "b": "Models of 1.5M to 34.5M parameters are easy to retrain on Indonesian scripts and fonts.", "c": "University units publish detectors and recognisers often.", "d": "Small OCR models fit eKYC and form pilots on modest hardware." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A releases OCR systems; Tier B and Tier C add Indonesian text data; Tier D builds document capture pilots.", "sources": [ "https://arxiv.org/abs/2606.13108" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "An End-to-End Trainable Neural Network for Image-based Sequence Recognition and Its Application to Scene Text Recognition", "authors": "Baoguang Shi et al.", "year": 2015, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1507.05717", "role": "founding" }, { "title": "Scene Text Recognition with Permuted Autoregressive Sequence Models", "authors": "Darwin Bautista et al.", "year": 2022, "venue": "ECCV 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2207.06966", "role": "state of the art" }, { "title": "TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models", "authors": "Minghao Li et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2109.10282", "role": "state of the art" }, { "title": "Character Region Awareness for Text Detection", "authors": "Youngmin Baek et al.", "year": 2019, "venue": "CVPR 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1904.01941", "role": "state of the art" }, { "title": "Real-time Scene Text Detection with Differentiable Binarization", "authors": "Minghui Liao et al.", "year": 2019, "venue": "AAAI 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/1911.08947", "role": "state of the art" } ], "approved": true }, { "id": "document-parsing-vision-language-models", "name": "Document parsing vision-language models", "definition": "Compact vision-language models that read whole pages into text, tables, formulas and charts in reading order.", "compute_anchor": "PaddleOCR-VL-0.9B combines a NaViT-style encoder with the ERNIE-4.5-0.3B language model.", "data_anchor": "Supports 109 languages; training data size is not in the abstract.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 18 authors on the PaddleOCR-VL paper (arXiv 2510.14528).", "time_horizon": "Months: OmniDocBench leaders change several times a year.", "shelf_life_signals": [ "General multimodal LLMs match dedicated parsers", "Indonesian documents enter public benchmarks" ], "exemplars": [ "baidu-paddleocr-vl", "deepseek-deepseek-ocr", "opendatalab-mineru2-5", "allen-institute-for-ai-olmocr" ], "observed_actors": [ "baidu", "deepseek", "opendatalab", "allen-ai" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Big tech labs release document VLMs.", "b": "Fine-tuning a 0.9B model is within an Indonesian lab budget, as Grab did with 1B and 2B models for Southeast Asian IDs.", "c": "A university unit can fine-tune and benchmark sub-1B models.", "d": "Document parsing feeds RAG and form extraction pilots directly." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A releases parsers; Tier B fine-tunes them for Indonesian documents; Tier D uses them in extraction and RAG pilots.", "sources": [ "https://arxiv.org/abs/2510.14528", "https://engineering.grab.com/custom-vision-llm-at-grab" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "OCR-free Document Understanding Transformer", "authors": "Geewook Kim et al.", "year": 2021, "venue": "ECCV 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2111.15664", "role": "founding" }, { "title": "olmOCR: Unlocking Trillions of Tokens in PDFs with Vision Language Models", "authors": "Jake Poznanski et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2502.18443", "role": "state of the art" }, { "title": "General OCR Theory: Towards OCR-2.0 via a Unified End-to-end Model", "authors": "Haoran Wei et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2409.01704", "role": "state of the art" }, { "title": "MinerU: An Open-Source Solution for Precise Document Content Extraction", "authors": "Bin Wang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2409.18839", "role": "state of the art" }, { "title": "Nougat: Neural Optical Understanding for Academic Documents", "authors": "Lukas Blecher et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2308.13418", "role": "state of the art" } ], "approved": true }, { "id": "document-layout-and-key-information-extraction", "name": "Document layout, table structure and key information extraction", "definition": "Models that use text, layout and image together to classify documents, find tables and pull fields from forms.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on the LayoutLMv3 paper (arXiv 2204.08387).", "time_horizon": "Years.", "shelf_life_signals": [ "Document VLMs absorb layout models", "Multilingual layout models cover Bahasa Indonesia forms" ], "exemplars": [ "microsoft-layoutlmv3", "microsoft-table-transformer", "baidu-pp-doclayoutv3", "naver-donut" ], "observed_actors": [ "microsoft", "baidu", "naver" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Microsoft and other big labs set the LayoutLM line.", "b": "Fine-tuning LayoutLM-class models on Indonesian forms fits an Indonesian lab.", "c": "A 5-author team produced LayoutLMv3; a university unit can work on it.", "d": "Field extraction from invoices and claims is a common proof of value." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A releases layout models; Tier C builds local form datasets; Tier D builds invoice and claims extraction pilots.", "sources": [ "https://arxiv.org/abs/2204.08387" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "LayoutLM: Pre-training of Text and Layout for Document Image Understanding", "authors": "Yiheng Xu et al.", "year": 2019, "venue": "KDD 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/1912.13318", "role": "founding" }, { "title": "FUNSD: A Dataset for Form Understanding in Noisy Scanned Documents", "authors": "Guillaume Jaume et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1905.13538", "role": "founding" }, { "title": "LayoutLMv3: Pre-training for Document AI with Unified Text and Image Masking", "authors": "Yupan Huang et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2204.08387", "role": "state of the art" }, { "title": "DocLayNet: A Large Human-Annotated Dataset for Document-Layout Analysis", "authors": "Birgit Pfitzmann et al.", "year": 2022, "venue": "KDD 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2206.01062", "role": "state of the art" }, { "title": "PubLayNet: largest dataset ever for document layout analysis", "authors": "Xu Zhong et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1908.07836", "role": "state of the art" } ], "approved": true }, { "id": "text-to-image-foundation-models", "name": "Text-to-image foundation models (diffusion and flow transformers)", "definition": "Large models that generate and edit images from text prompts.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; Qwen-Image trains on text-to-image, text-image-to-image and image-to-image reconstruction tasks.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 39 authors on the Qwen-Image report (arXiv 2508.02324).", "time_horizon": "Months: leaderboards change monthly.", "shelf_life_signals": [ "A new open-weight model tops the arena", "Licensing and copyright rulings restrict training data" ], "exemplars": [ "alibaba-qwen-image", "black-forest-labs-flux-1-dev", "stability-ai-stable-diffusion-3-5", "google-nano-banana-pro" ], "observed_actors": [ "alibaba", "black-forest-labs", "stability-ai", "google" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Frontier and big-tech labs train these models with large teams, for example 39 authors on Qwen-Image.", "b": "Pretraining a competitive model is not feasible for an Indonesian lab.", "c": "A university unit can fine-tune open weights, for example with LoRA.", "d": "Released models and APIs support marketing and design proofs of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A trains and releases models; Tier C fine-tunes for local styles; Tier D uses APIs and open weights in content pilots.", "sources": [ "https://arxiv.org/abs/2508.02324" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "High-Resolution Image Synthesis with Latent Diffusion Models", "authors": "Robin Rombach et al.", "year": 2021, "venue": "CVPR 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2112.10752", "role": "founding" }, { "title": "FLUX.1 Kontext: Flow Matching for In-Context Image Generation and Editing in Latent Space", "authors": "Black Forest Labs et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2506.15742", "role": "state of the art" }, { "title": "Scaling Rectified Flow Transformers for High-Resolution Image Synthesis", "authors": "Patrick Esser et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.03206", "role": "state of the art" }, { "title": "SDXL: Improving Latent Diffusion Models for High-Resolution Image Synthesis", "authors": "Dustin Podell et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2307.01952", "role": "state of the art" }, { "title": "Hierarchical Text-Conditional Image Generation with CLIP Latents", "authors": "Aditya Ramesh et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2204.06125", "role": "state of the art" } ], "approved": true }, { "id": "unified-multimodal-understanding-and-generation", "name": "Unified multimodal models for understanding and generation", "definition": "Single models that both read images and generate them, so editing and reasoning share one backbone.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; BAGEL reports free-form image manipulation and future frame prediction.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 12 authors on the BAGEL paper (arXiv 2505.14683).", "time_horizon": "Year.", "shelf_life_signals": [ "Frontier chat models ship native image output", "Open unified models match specialist generators" ], "exemplars": [ "bagel-7b-mot", "emu3-5", "blip3-o-8b", "tencent-hunyuanimage-3-0" ], "observed_actors": [ "bytedance", "baai", "salesforce", "tencent" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "stretch", "tier_rationale": { "a": "Frontier labs ship native image generation inside multimodal models.", "b": "Pretraining unified models is not feasible for an Indonesian lab.", "c": "A university unit can study and fine-tune 7B to 8B open unified models.", "d": "Delivery teams use them through APIs; building on them is a stretch." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A trains unified models; Tier C studies them; Tier D uses them via APIs for editing workflows.", "sources": [ "https://arxiv.org/abs/2505.14683" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "Chameleon: Mixed-Modal Early-Fusion Foundation Models", "authors": "Chameleon Team", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2405.09818", "role": "founding" }, { "title": "Emerging Properties in Unified Multimodal Pretraining", "authors": "Chaorui Deng et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2505.14683", "role": "state of the art" }, { "title": "Transfusion: Predict the Next Token and Diffuse Images with One Multi-Modal Model", "authors": "Chunting Zhou et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2408.11039", "role": "state of the art" }, { "title": "Show-o: One Single Transformer to Unify Multimodal Understanding and Generation", "authors": "Jinheng Xie et al.", "year": 2024, "venue": "ICLR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2408.12528", "role": "state of the art" }, { "title": "Janus: Decoupling Visual Encoding for Unified Multimodal Understanding and Generation", "authors": "Chengyue Wu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2410.13848", "role": "state of the art" } ], "approved": true }, { "id": "few-step-diffusion-distillation", "name": "Few-step distillation of diffusion models", "definition": "Compressing slow diffusion generators into models that produce an image or video in one to eight steps.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; SDXL-Lightning distils SDXL for one-step and few-step 1024px generation.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on the SDXL-Lightning paper (arXiv 2402.13929).", "time_horizon": "Months.", "shelf_life_signals": [ "One-step generation matches full-step quality", "Distilled turbo variants ship with every major release" ], "exemplars": [ "bytedance-sdxl-lightning", "latent-consistency-model", "stability-ai-sd-turbo", "black-forest-labs-flux-1-schnell" ], "observed_actors": [ "bytedance", "tsinghua-university", "stability-ai", "black-forest-labs" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Big labs release distilled variants with each model.", "b": "Distilling an open model needs far less compute than pretraining it, so it fits an Indonesian lab.", "c": "A 3-author team produced SDXL-Lightning; a university unit can do this.", "d": "Few-step models cut serving cost in generation pilots." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A releases distilled models; Tier C studies distillation; Tier D uses fast variants to cut cost in pilots.", "sources": [ "https://arxiv.org/abs/2402.13929" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "Progressive Distillation for Fast Sampling of Diffusion Models", "authors": "Tim Salimans et al.", "year": 2022, "venue": "ICLR 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2202.00512", "role": "founding" }, { "title": "Consistency Models", "authors": "Yang Song et al.", "year": 2023, "venue": "ICML 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2303.01469", "role": "founding" }, { "title": "Improved Distribution Matching Distillation for Fast Image Synthesis", "authors": "Tianwei Yin et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2405.14867", "role": "state of the art" }, { "title": "Latent Consistency Models: Synthesizing High-Resolution Images with Few-Step Inference", "authors": "Simian Luo et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2310.04378", "role": "state of the art" }, { "title": "Adversarial Diffusion Distillation", "authors": "Axel Sauer et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2311.17042", "role": "state of the art" } ], "approved": true }, { "id": "controllable-and-personalised-image-generation", "name": "Controllable and personalised image generation", "definition": "Adding spatial controls such as edges, depth and pose, or identity and style references, to text-to-image models.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "ControlNet training is robust with small (under 50k) and large (over 1M) datasets.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on the ControlNet paper (arXiv 2302.05543).", "time_horizon": "Months to a year.", "shelf_life_signals": [ "Base models accept reference images natively", "Identity tools face consent and deepfake rules" ], "exemplars": [ "controlnet-family", "ip-adapter", "instantid", "tencent-photomaker" ], "observed_actors": [ "stanford-university", "tencent" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Big labs build reference control into new models.", "b": "Training a ControlNet on under 50k images fits an Indonesian lab.", "c": "A 3-author team produced ControlNet; a university unit can work on it.", "d": "Brand-consistent product and layout generation is a direct proof of value." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and academia release control methods; Tier C trains local controls; Tier D builds brand-consistent content pilots.", "sources": [ "https://arxiv.org/abs/2302.05543" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "An Image is Worth One Word: Personalizing Text-to-Image Generation using Textual Inversion", "authors": "Rinon Gal et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2208.01618", "role": "founding" }, { "title": "DreamBooth: Fine Tuning Text-to-Image Diffusion Models for Subject-Driven Generation", "authors": "Nataniel Ruiz et al.", "year": 2022, "venue": "CVPR 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2208.12242", "role": "founding" }, { "title": "Adding Conditional Control to Text-to-Image Diffusion Models", "authors": "Lvmin Zhang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2302.05543", "role": "state of the art" }, { "title": "T2I-Adapter: Learning Adapters to Dig out More Controllable Ability for Text-to-Image Diffusion Models", "authors": "Chong Mou et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2302.08453", "role": "state of the art" }, { "title": "IP-Adapter: Text Compatible Image Prompt Adapter for Text-to-Image Diffusion Models", "authors": "Hu Ye et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2308.06721", "role": "state of the art" } ], "approved": true }, { "id": "instruction-based-image-editing-and-inpainting", "name": "Instruction-based image editing and inpainting", "definition": "Changing, removing or filling parts of an image from a written instruction or a mask.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Training data is generated with GPT-3 and Stable Diffusion; size not in the abstract.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on the InstructPix2Pix paper (arXiv 2211.09800).", "time_horizon": "Months.", "shelf_life_signals": [ "General image models edit as well as specialist editors", "Editing provenance labels become mandatory" ], "exemplars": [ "instruct-pix2pix", "black-forest-labs-flux-1-kontext", "lama-inpainting", "stepfun-step1x-edit" ], "observed_actors": [ "uc-berkeley", "black-forest-labs", "samsung", "skoltech", "stepfun" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Big labs ship editing models such as FLUX Kontext.", "b": "Building synthetic edit data from open models, as InstructPix2Pix did, fits an Indonesian lab.", "c": "A 3-author team produced InstructPix2Pix; a university unit can work on it.", "d": "Product photo editing and virtual try-on give fast retail proofs of value." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A releases editors; Tier C builds local edit datasets; Tier D uses editors in retail and media pilots.", "sources": [ "https://arxiv.org/abs/2211.09800" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "SDEdit: Guided Image Synthesis and Editing with Stochastic Differential Equations", "authors": "Chenlin Meng et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2108.01073", "role": "founding" }, { "title": "FLUX.1 Kontext: Flow Matching for In-Context Image Generation and Editing in Latent Space", "authors": "Black Forest Labs et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2506.15742", "role": "state of the art" }, { "title": "MagicBrush: A Manually Annotated Dataset for Instruction-Guided Image Editing", "authors": "Kai Zhang et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2306.10012", "role": "state of the art" }, { "title": "RePaint: Inpainting using Denoising Diffusion Probabilistic Models", "authors": "Andreas Lugmayr et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2201.09865", "role": "state of the art" }, { "title": "Prompt-to-Prompt Image Editing with Cross Attention Control", "authors": "Amir Hertz et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2208.01626", "role": "state of the art" } ], "approved": true }, { "id": "generative-image-modelling-methods", "name": "Generative image modelling methods (GANs, diffusion and image translation)", "definition": "Core methods for learning to generate or translate images, from GANs and autoencoders to latent diffusion.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; latent diffusion runs diffusion in the latent space of a pretrained autoencoder.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on the latent diffusion paper (arXiv 2112.10752).", "time_horizon": "Years.", "shelf_life_signals": [ "Flow matching replaces diffusion training", "Autoregressive image tokens compete with diffusion" ], "exemplars": [ "latent-diffusion-ldm", "nvidia-stylegan2", "cyclegan", "ddpm" ], "observed_actors": [ "lmu-munich", "runway", "nvidia", "uc-berkeley" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Frontier labs build on these methods at scale.", "b": "Training GANs and small diffusion models fits an Indonesian lab.", "c": "Small academic teams wrote most of these methods, such as the 5-author latent diffusion paper.", "d": "Delivery teams use finished models rather than new methods; method work is a stretch." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "medium" }, "pipeline_role": "Tier C and academia invent methods; Tier A scales them; Tier D uses the resulting models, for example for synthetic training data.", "sources": [ "https://arxiv.org/abs/2112.10752" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "Denoising Diffusion Probabilistic Models", "authors": "Jonathan Ho et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2006.11239", "role": "founding" }, { "title": "Score-Based Generative Modeling through Stochastic Differential Equations", "authors": "Yang Song et al.", "year": 2020, "venue": "ICLR 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2011.13456", "role": "founding" }, { "title": "Elucidating the Design Space of Diffusion-Based Generative Models", "authors": "Tero Karras et al.", "year": 2022, "venue": "NeurIPS 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2206.00364", "role": "state of the art" }, { "title": "Scalable Diffusion Models with Transformers", "authors": "William Peebles et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2212.09748", "role": "state of the art" } ], "approved": true }, { "id": "generative-media-licensing-and-provenance", "name": "Training-data licensing and content provenance for generative media", "definition": "Building image, video and avatar generators on licensed data and marking their outputs with provenance metadata or watermarks.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Adobe Firefly is trained on 300 million images and videos in Adobe Stock plus public domain and openly licensed sources.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): not in the sources read.", "time_horizon": "Year: tied to court cases and AI law.", "shelf_life_signals": [ "Court rulings on training data", "C2PA or watermark mandates in AI law" ], "exemplars": [ "adobe-firefly-image-5", "bria-ai-bria", "bria-ai-fibo" ], "observed_actors": [ "adobe", "bria-ai" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Firms such as Adobe with 300 million licensed stock assets can train licensed-only models.", "b": "An Indonesian lab rarely owns licensed media at that scale.", "c": "Provenance and watermark studies fit a university unit.", "d": "Choosing licensed models and C2PA labelling is part of every client generative media pilot." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A and stock owners train licensed models; Tier C studies provenance; Tier D chooses compliant models and labels outputs in pilots.", "sources": [ "https://en.wikipedia.org/wiki/Adobe_Firefly" ], "unverified": [ "compute_anchor", "team_size_proxy" ], "papers": [ { "title": "Extracting Training Data from Diffusion Models", "authors": "Nicholas Carlini et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2301.13188", "role": "founding" }, { "title": "The Stable Signature: Rooting Watermarks in Latent Diffusion Models", "authors": "Pierre Fernandez et al.", "year": 2023, "venue": "ICCV 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2303.15435", "role": "state of the art" }, { "title": "Tree-Ring Watermarks: Fingerprints for Diffusion Images that are Invisible and Robust", "authors": "Yuxin Wen et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.20030", "role": "state of the art" }, { "title": "Glaze: Protecting Artists from Style Mimicry by Text-to-Image Models", "authors": "Shawn Shan et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2302.04222", "role": "state of the art" }, { "title": "Foundation Models and Fair Use", "authors": "Peter Henderson et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2303.15715", "role": "survey" } ], "approved": true }, { "id": "text-to-video-foundation-models", "name": "Text-to-video foundation models", "definition": "Large models that generate video clips from text or images.", "compute_anchor": "Wan offers 1.3B and 14B models; the 1.3B model needs only 8.19 GB VRAM. Training compute is not stated in the abstract.", "data_anchor": "The 14B model is trained on billions of images and videos.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 62 authors on the Wan paper (arXiv 2503.20314).", "time_horizon": "Months.", "shelf_life_signals": [ "A new open model tops the video arenas", "Copyright actions against video models" ], "exemplars": [ "alibaba-wan-2-1", "google-deepmind-veo-3", "kuaishou-kling-3-0", "tencent-hunyuanvideo-1-5" ], "observed_actors": [ "alibaba", "google-deepmind", "kuaishou", "tencent" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Training on billions of images and videos with a 62-author team is frontier-lab work.", "b": "Pretraining at that scale is not feasible for an Indonesian lab.", "c": "A university unit can fine-tune the 1.3B model, which runs in 8.19 GB VRAM.", "d": "APIs and the small open model support marketing and training video pilots." }, "tier_confidence": { "a": "high", "b": "high", "c": "medium", "d": "high" }, "pipeline_role": "Tier A trains video models; Tier C fine-tunes small open ones; Tier D uses them in content pilots.", "sources": [ "https://arxiv.org/abs/2503.20314" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Imagen Video: High Definition Video Generation with Diffusion Models", "authors": "Jonathan Ho et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2210.02303", "role": "founding" }, { "title": "Wan: Open and Advanced Large-Scale Video Generative Models", "authors": "Team Wan et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2503.20314", "role": "state of the art" }, { "title": "CogVideoX: Text-to-Video Diffusion Models with An Expert Transformer", "authors": "Zhuoyi Yang et al.", "year": 2024, "venue": "ICLR2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2408.06072", "role": "state of the art" }, { "title": "Align your Latents: High-Resolution Video Synthesis with Latent Diffusion Models", "authors": "Andreas Blattmann et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2304.08818", "role": "state of the art" }, { "title": "Video Diffusion Models: A Survey", "authors": "Andrew Melnik et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2405.03150", "role": "survey" } ], "approved": true }, { "id": "joint-audio-video-generation", "name": "Joint audio and video generation", "definition": "Generating video together with synchronised speech, sound effects and music in one model.", "compute_anchor": "Ovi uses twin backbones with a 5B audio branch pretrained from scratch; the released checkpoint is 11B.", "data_anchor": "In-house audio datasets per the model card; size not given.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on the Ovi paper (arXiv 2510.01284).", "time_horizon": "Months.", "shelf_life_signals": [ "Native audio becomes standard in every video model", "Voice cloning rules restrict generated speech" ], "exemplars": [ "character-ai-ovi", "lightricks-ltx-2-5", "openmoss-mova", "kuaishou-kling-2-6" ], "observed_actors": [ "character-ai", "lightricks", "openmoss", "kuaishou" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs ship native audio in video models.", "b": "Pretraining 11B audio-video models is not feasible for an Indonesian lab.", "c": "A university unit can study the open models.", "d": "Videos with Indonesian narration are a direct content proof of value through APIs." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A trains joint models; Tier C studies open ones; Tier D uses them for narrated content pilots.", "sources": [ "https://arxiv.org/abs/2510.01284", "https://huggingface.co/chetwinlow1/Ovi" ], "unverified": [ "data_anchor" ], "papers": [ { "title": "MM-Diffusion: Learning Multi-Modal Diffusion Models for Joint Audio and Video Generation", "authors": "Ludan Ruan et al.", "year": 2022, "venue": "CVPR 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2212.09478", "role": "founding" }, { "title": "JavisDiT: Joint Audio-Video Diffusion Transformer with Hierarchical Spatio-Temporal Prior Synchronization", "authors": "Kai Liu et al.", "year": 2025, "venue": "ICLR 2026 (per arXiv comment)", "url": "https://arxiv.org/abs/2503.23377", "role": "state of the art" }, { "title": "UniVerse-1: Unified Audio-Video Generation via Stitching of Experts", "authors": "Duomin Wang et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2509.06155", "role": "state of the art" }, { "title": "Ovi: Twin Backbone Cross-Modal Fusion for Audio-Video Generation", "authors": "Chetwin Low et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2510.01284", "role": "state of the art" }, { "title": "AV-DiT: Efficient Audio-Visual Diffusion Transformer for Joint Audio and Video Generation", "authors": "Kai Wang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.07686", "role": "state of the art" } ], "approved": true }, { "id": "video-editing-and-controllable-video-generation", "name": "Video editing and controllable video generation", "definition": "Editing existing video, or steering generation with references, masks, poses or motion modules.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; VACE unifies reference-to-video, video-to-video and masked editing.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on the VACE paper (arXiv 2503.07598).", "time_horizon": "Months.", "shelf_life_signals": [ "General video models accept edit instructions", "Real-time video editing reaches consumer devices" ], "exemplars": [ "alibaba-wan-2-1-vace", "decart-lucy-edit-dev", "netflix-void", "animatediff" ], "observed_actors": [ "alibaba", "decart", "netflix", "shanghai-ai-lab", "cuhk", "stanford-university" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Big labs release editing models built on their video bases.", "b": "Fine-tuning editing models needs a video base model, a stretch for an Indonesian lab.", "c": "A 6-author team produced VACE; a university unit can work on adapters.", "d": "Video localisation and object removal are clear media pilots." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A releases editing models; Tier C builds adapters; Tier D uses them in media pilots.", "sources": [ "https://arxiv.org/abs/2503.07598" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "Tune-A-Video: One-Shot Tuning of Image Diffusion Models for Text-to-Video Generation", "authors": "Jay Zhangjie Wu et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2212.11565", "role": "founding" }, { "title": "VACE: All-in-One Video Creation and Editing", "authors": "Zeyinzi Jiang et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2503.07598", "role": "state of the art" }, { "title": "Text2Video-Zero: Text-to-Image Diffusion Models are Zero-Shot Video Generators", "authors": "Levon Khachatryan et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2303.13439", "role": "state of the art" }, { "title": "AnimateDiff: Animate Your Personalized Text-to-Image Diffusion Models without Specific Tuning", "authors": "Yuwei Guo et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2307.04725", "role": "state of the art" }, { "title": "A Survey on Video Diffusion Models", "authors": "Zhen Xing et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2310.10647", "role": "survey" } ], "approved": true }, { "id": "audio-driven-talking-heads-and-lip-sync", "name": "Audio-driven talking heads and lip-sync", "definition": "Generating a talking person video from one photo and an audio track, or re-syncing lips to new audio for dubbing.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; OmniHuman-1 mixes motion-related conditions into training to scale data.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on the OmniHuman-1 paper (arXiv 2502.01061).", "time_horizon": "Months.", "shelf_life_signals": [ "Real-time interactive avatars", "Deepfake fraud drives consent and watermark rules" ], "exemplars": [ "bytedance-omnihuman-1", "sadtalker", "bytedance-latentsync-1-6", "nvidia-audio2face-3d" ], "observed_actors": [ "bytedance", "xi-an-jiaotong-university", "tencent", "nvidia" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Big tech trains human video models on large data.", "b": "Training is a stretch for an Indonesian lab; open models such as LatentSync can be fine-tuned.", "c": "University units publish talking-head methods such as SadTalker.", "d": "Avatars and dubbing in Bahasa Indonesia are clear pilots, with consent checks." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A trains avatar models; Tier C adapts open ones; Tier D builds dubbing and avatar pilots with consent controls.", "sources": [ "https://arxiv.org/abs/2502.01061" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "A Lip Sync Expert Is All You Need for Speech to Lip Generation In The Wild", "authors": "K R Prajwal et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2008.10010", "role": "founding" }, { "title": "EMO: Emote Portrait Alive -- Generating Expressive Portrait Videos with Audio2Video Diffusion Model under Weak Conditions", "authors": "Linrui Tian et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.17485", "role": "state of the art" }, { "title": "Hallo: Hierarchical Audio-Driven Visual Synthesis for Portrait Image Animation", "authors": "Mingwang Xu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.08801", "role": "state of the art" }, { "title": "LivePortrait: Efficient Portrait Animation with Stitching and Retargeting Control", "authors": "Jianzhu Guo et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2407.03168", "role": "state of the art" }, { "title": "SadTalker: Learning Realistic 3D Motion Coefficients for Stylized Audio-Driven Single Image Talking Face Animation", "authors": "Wenxuan Zhang et al.", "year": 2022, "venue": "CVPR 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2211.12194", "role": "state of the art" } ], "approved": true }, { "id": "reference-driven-human-image-animation", "name": "Reference-driven human image animation", "definition": "Animating a person or character in a still image to follow the motion of a driving video or pose sequence.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; DreamActor-M1 uses implicit face features, 3D head spheres and 3D body skeletons as guidance.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on the DreamActor-M1 paper (arXiv 2504.01724).", "time_horizon": "Months.", "shelf_life_signals": [ "General video models absorb animation control", "Identity misuse rules" ], "exemplars": [ "bytedance-dreamactor-m1", "alibaba-unianimate", "first-order-motion-model", "alibaba-wan2-2-animate-2-14b" ], "observed_actors": [ "bytedance", "alibaba", "university-of-trento", "snap" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Big tech releases DiT-based animation models.", "b": "Training is a stretch for an Indonesian lab.", "c": "A 6-author team produced DreamActor-M1; a university unit can work on methods.", "d": "Client value is mostly in marketing content, a narrower pilot." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A trains animation models; Tier C studies them; Tier D uses them in content pilots.", "sources": [ "https://arxiv.org/abs/2504.01724" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "DisCo: Disentangled Control for Realistic Human Dance Generation", "authors": "Tan Wang et al.", "year": 2023, "venue": "CVPR24 (per arXiv comment)", "url": "https://arxiv.org/abs/2307.00040", "role": "founding" }, { "title": "Champ: Controllable and Consistent Human Image Animation with 3D Parametric Guidance", "authors": "Shenhao Zhu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.14781", "role": "state of the art" }, { "title": "UniAnimate: Taming Unified Video Diffusion Models for Consistent Human Image Animation", "authors": "Xiang Wang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.01188", "role": "state of the art" }, { "title": "Animate Anyone: Consistent and Controllable Image-to-Video Synthesis for Character Animation", "authors": "Li Hu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2311.17117", "role": "state of the art" }, { "title": "MagicAnimate: Temporally Consistent Human Image Animation using Diffusion Model", "authors": "Zhongcong Xu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2311.16498", "role": "state of the art" } ], "approved": true }, { "id": "image-to-3d-asset-generation", "name": "Image-to-3D and text-to-3D asset generation", "definition": "Generating textured 3D meshes from one or a few images or a text prompt.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; Hunyuan3D 2.0 has a shape model (Hunyuan3D-DiT) and a texture model (Hunyuan3D-Paint).", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 74 authors on the Hunyuan3D 2.0 report (arXiv 2501.12202).", "time_horizon": "Months.", "shelf_life_signals": [ "Production-ready PBR meshes from one image", "Generated 3D scenes rather than single objects" ], "exemplars": [ "tencent-hunyuan3d-2-0", "microsoft-trellis", "stability-ai-triposr", "tencent-instantmesh" ], "observed_actors": [ "tencent", "microsoft", "stability-ai", "vast" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "stretch", "tier_rationale": { "a": "A 74-author team built Hunyuan3D 2.0, frontier-lab scale.", "b": "Pretraining is not feasible for an Indonesian lab.", "c": "A university unit can fine-tune small open models such as TripoSR.", "d": "Product 3D views and digital twins are possible pilots, but output quality needs checking." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A releases 3D generators; Tier C tests them; Tier D uses them for product and asset previews.", "sources": [ "https://arxiv.org/abs/2501.12202" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "DreamFusion: Text-to-3D using 2D Diffusion", "authors": "Ben Poole et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2209.14988", "role": "founding" }, { "title": "Hunyuan3D 2.0: Scaling Diffusion Models for High Resolution Textured 3D Assets Generation", "authors": "Zibo Zhao et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2501.12202", "role": "state of the art" }, { "title": "Structured 3D Latents for Scalable and Versatile 3D Generation", "authors": "Jianfeng Xiang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2412.01506", "role": "state of the art" }, { "title": "Zero-1-to-3: Zero-shot One Image to 3D Object", "authors": "Ruoshi Liu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2303.11328", "role": "state of the art" }, { "title": "LRM: Large Reconstruction Model for Single Image to 3D", "authors": "Yicong Hong et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2311.04400", "role": "state of the art" } ], "approved": true }, { "id": "optical-flow-and-local-feature-matching", "name": "Optical flow and local feature matching", "definition": "Learning where each pixel or keypoint in one image moves to in another image, the dense and sparse correspondence that video, mapping and 3D pipelines depend on.", "compute_anchor": "Training compute is not in the sources read; the RAFT paper states only that it is efficient in inference time, training speed and parameter count.", "data_anchor": "SuperPoint is trained on the MS-COCO generic image dataset with Homographic Adaptation; RAFT reports a KITTI F1-all error of 5.10 percent and a Sintel final-pass end-point error of 2.855 pixels.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 2 authors on the RAFT paper (arXiv 2003.12039) and 3 on LightGlue (arXiv 2306.13643).", "time_horizon": "Months; small teams publish new matchers and flow networks within one conference cycle.", "shelf_life_signals": [ "Feed-forward 3D models that output matches and tracks directly replace separate flow and matching stages", "LightGlue adapts its compute to how hard an image pair is, opening latency-sensitive uses" ], "exemplars": [ "raft", "lightglue", "superpoint", "loftr" ], "observed_actors": [ "princeton-university", "eth-zurich", "magic-leap", "zhejiang-university", "university-of-freiburg" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs can run this work easily, though the exemplars come from universities and Magic Leap rather than frontier labs.", "b": "RAFT came from a 2-author team, a size an Indonesian lab can field for new correspondence models.", "c": "SuperPoint trains on the public MS-COCO set and RAFT on public flow benchmarks, so a university unit can reproduce and extend them.", "d": "Released matchers such as LightGlue are meant for latency-sensitive 3D reconstruction, which suits survey and inspection proofs of concept." }, "tier_confidence": { "a": "medium", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier C and university labs publish flow and matching networks; Tier B adapts them to local imagery; Tier D builds them into mapping, video stabilisation and inspection proofs of concept, and Tier A folds them into feed-forward 3D models.", "sources": [ "https://arxiv.org/abs/2003.12039", "https://arxiv.org/abs/2306.13643", "https://arxiv.org/abs/1712.07629" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "FlowNet: Learning Optical Flow with Convolutional Networks", "authors": "Philipp Fischer et al.", "year": 2015, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1504.06852", "role": "founding" }, { "title": "SuperPoint: Self-Supervised Interest Point Detection and Description", "authors": "Daniel DeTone et al.", "year": 2017, "venue": "CVPR 2018 (per arXiv comment)", "url": "https://arxiv.org/abs/1712.07629", "role": "founding" }, { "title": "RoMa: Robust Dense Feature Matching", "authors": "Johan Edstedt et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.15404", "role": "state of the art" }, { "title": "LightGlue: Local Feature Matching at Light Speed", "authors": "Philipp Lindenberger et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.13643", "role": "state of the art" }, { "title": "LoFTR: Detector-Free Local Feature Matching with Transformers", "authors": "Jiaming Sun et al.", "year": 2021, "venue": "CVPR 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2104.00680", "role": "state of the art" } ], "approved": true }, { "id": "neural-radiance-fields-and-gaussian-splatting", "name": "Neural radiance fields and Gaussian splatting", "definition": "Optimising a 3D scene representation, a neural radiance field or a set of 3D Gaussians, from posed photos so that new views can be rendered photorealistically.", "compute_anchor": "Training compute is not in the sources read; Zip-NeRF trains 24 times faster than mip-NeRF 360, and 3D Gaussian Splatting renders at 30 frames per second or more at 1080p.", "data_anchor": "NeRF needs only a set of images with known camera poses of one scene; no large training corpus is involved for per-scene optimisation.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on the NeRF paper (arXiv 2003.08934), 4 on 3D Gaussian Splatting (arXiv 2308.04079) and 5 on Zip-NeRF (arXiv 2304.06706).", "time_horizon": "Months to a year.", "shelf_life_signals": [ "Gaussian splatting gave real-time rendering that grid-based NeRFs did not", "Feed-forward models such as Apple SHARP build a Gaussian scene from one image in under a second, removing per-scene optimisation" ], "exemplars": [ "google-zip-nerf", "google-camp", "apple-sharp" ], "observed_actors": [ "google", "apple", "inria" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google publishes Zip-NeRF and CamP and Apple publishes SHARP, so frontier labs lead the method work.", "b": "Per-scene optimisation needs only posed photos of one scene, which an Indonesian lab can capture and process.", "c": "The 4-author Gaussian Splatting paper shows a small university team can set the state of the art here.", "d": "Real-time rendering at 1080p lets a delivery team turn site photos into navigable digital twins for proofs of value." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and Tier C publish scene representations; Tier B captures local sites; Tier D turns them into digital twin and inspection proofs of concept.", "sources": [ "https://arxiv.org/abs/2003.08934", "https://arxiv.org/abs/2308.04079", "https://arxiv.org/abs/2304.06706" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "NeRF: Representing Scenes as Neural Radiance Fields for View Synthesis", "authors": "Ben Mildenhall et al.", "year": 2020, "venue": "ECCV 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2003.08934", "role": "founding" }, { "title": "2D Gaussian Splatting for Geometrically Accurate Radiance Fields", "authors": "Binbin Huang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.17888", "role": "state of the art" }, { "title": "3D Gaussian Splatting for Real-Time Radiance Field Rendering", "authors": "Bernhard Kerbl et al.", "year": 2023, "venue": "ACM Transactions on Graphics, volume 42(4), July 2023", "url": "https://arxiv.org/abs/2308.04079", "role": "state of the art" }, { "title": "Instant Neural Graphics Primitives with a Multiresolution Hash Encoding", "authors": "Thomas Müller et al.", "year": 2022, "venue": "ACM Trans. Graph. 41, 4, Article 102 (July 2022), 15 pages", "url": "https://arxiv.org/abs/2201.05989", "role": "state of the art" }, { "title": "NeRF: Neural Radiance Field in 3D Vision: A Comprehensive Review (Updated Post-Gaussian Splatting)", "authors": "Kyle Gao et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2210.00379", "role": "survey" } ], "approved": true }, { "id": "image-based-virtual-try-on", "name": "Image-based virtual try-on", "definition": "Generating a photo of a given person wearing a given garment while keeping the garment's texture, logo and fit, for online fashion retail.", "compute_anchor": "Training compute is not in the sources read.", "data_anchor": "Not in the sources read; IDM-VTON takes a pair of images, one of the person and one of the garment, and builds on a pretrained diffusion UNet.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on IDM-VTON (arXiv 2403.05139) and 6 on CP-VTON (arXiv 1807.07688).", "time_horizon": "Months.", "shelf_life_signals": [ "Methods moved from GAN warping (CP-VTON) to diffusion models that keep garment identity (IDM-VTON)", "General instruction-based image editors with reference images may absorb dedicated try-on models" ], "exemplars": [ "idm-vton", "cp-vton" ], "observed_actors": [ "kaist", "omnious", "sun-yat-sen-university", "sensetime" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs can do this, but the exemplars come from KAIST with OMNIOUS and from Sun Yat-sen University with SenseTime.", "b": "IDM-VTON adapts a pretrained diffusion model with a 5-author team, a scale an Indonesian lab can match with local garment data.", "c": "CP-VTON and IDM-VTON are university papers with 5 and 6 authors, the usual size of a university unit.", "d": "Released try-on models on a pretrained diffusion base let a delivery team show online retailers a try-on proof of value." }, "tier_confidence": { "a": "medium", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier C publishes try-on methods on Tier A diffusion bases; Tier B fits them to Indonesian clothing and body types; Tier D runs retail proofs of value.", "sources": [ "https://arxiv.org/abs/2403.05139", "https://arxiv.org/abs/1807.07688" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "VITON: An Image-based Virtual Try-on Network", "authors": "Xintong Han et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1711.08447", "role": "founding" }, { "title": "OOTDiffusion: Outfitting Fusion based Latent Diffusion for Controllable Virtual Try-on", "authors": "Yuhao Xu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.01779", "role": "state of the art" }, { "title": "Improving Diffusion Models for Authentic Virtual Try-on in the Wild", "authors": "Yisol Choi et al.", "year": 2024, "venue": "ECCV 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2403.05139", "role": "state of the art" }, { "title": "CatVTON: Concatenation Is All You Need for Virtual Try-On with Diffusion Models", "authors": "Zheng Chong et al.", "year": 2024, "venue": "ICLR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2407.15886", "role": "state of the art" }, { "title": "TryOnDiffusion: A Tale of Two UNets", "authors": "Luyang Zhu et al.", "year": 2023, "venue": "CVPR 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2306.08276", "role": "state of the art" } ], "approved": true }, { "id": "massively-multilingual-speech-recognition", "name": "Massively multilingual speech recognition", "definition": "Training one speech recognition model on hundreds of thousands of hours or hundreds of languages so it transcribes many languages, often zero-shot.", "compute_anchor": "Omnilingual ASR scales self-supervised pre-training to 7B parameters and releases a family from 300M to 7B; Whisper and MMS abstracts give no training compute.", "data_anchor": "Whisper: 680,000 hours of multilingual and multitask supervision; MMS: one ASR model for 1,107 languages built from readings of religious texts; Omnilingual ASR: over 1,600 languages.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on Whisper; 33 listed on Omnilingual ASR.", "time_horizon": "Months to fine-tune a released model; years to build a new one.", "shelf_life_signals": [ "A single open model covering most languages with low error, leaving only fine-tuning", "Audio language models replacing dedicated recognisers" ], "exemplars": [ "openai-whisper", "meta-mms", "meta-omnilingual-asr", "carnegie-mellon-university-owsm-v4" ], "observed_actors": [ "openai", "meta", "nvidia", "alibaba", "carnegie-mellon-university", "ibm", "cohere" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "OpenAI and Meta trained on 680,000 hours (Whisper) and scaled pre-training to 7B parameters (Omnilingual ASR).", "b": "Training from scratch at 680,000 hours or 7B parameters is beyond a national lab, but fine-tuning the released 300M to 7B models is within reach.", "c": "The ITB and UI study fine-tuned Whisper and MMS on Indonesian rather than training a multilingual model, which is the realistic scope for a unit.", "d": "Released checkpoints from 300M parameters upward, trained on up to 680,000 hours, can be applied directly in a transcription proof of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A releases multilingual recognisers; Tier B and Tier C fine-tune and benchmark them on Indonesian and regional speech; Tier D deploys them in transcription proofs of value.", "sources": [ "https://arxiv.org/abs/2212.04356", "https://arxiv.org/abs/2305.13516", "https://arxiv.org/abs/2511.09690", "https://arxiv.org/abs/2410.08828" ], "unverified": [], "papers": [ { "title": "Robust Speech Recognition via Large-Scale Weak Supervision", "authors": "Alec Radford et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2212.04356", "role": "founding" }, { "title": "Scaling Speech Technology to 1,000+ Languages", "authors": "Vineel Pratap et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.13516", "role": "state of the art" }, { "title": "Google USM: Scaling Automatic Speech Recognition Beyond 100 Languages", "authors": "Yu Zhang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2303.01037", "role": "state of the art" }, { "title": "Reproducing Whisper-Style Training Using an Open-Source Toolkit and Publicly Available Data", "authors": "Yifan Peng et al.", "year": 2023, "venue": "ASRU 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2309.13876", "role": "state of the art" }, { "title": "XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale", "authors": "Arun Babu et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2111.09296", "role": "state of the art" } ], "approved": true }, { "id": "speech-recognition-indonesian-regional-languages", "name": "Speech recognition for Indonesian and regional languages", "definition": "Adapting speech recognition to Indonesian and regional languages such as Javanese and Sundanese, including spontaneous, informal and noisy speech.", "compute_anchor": "Not in the sources read: the IDSV study abstract gives no compute; it fine-tunes existing Whisper and MMS models.", "data_anchor": "IDSV study: Indonesian data is dominated by read, formal and clean speech, so the authors compiled a dataset with speaking-style, context and noise variability.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on the IDSV study (ITB, UI, JAIST, NAIST).", "time_horizon": "Months.", "shelf_life_signals": [ "Multilingual models reaching low error on spontaneous Indonesian without fine-tuning", "Public Indonesian corpora with spontaneous and noisy speech" ], "exemplars": [ "idsv-ft-whisper", "idsv-mms-ft-lm", "indonesian-nlp-wav2vec2-indonesian-javanese-sundanese" ], "observed_actors": [ "institut-teknologi-bandung", "universitas-indonesia", "jaist", "naist", "indonesian-nlp" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs cover Indonesian inside multilingual models such as Whisper and MMS, which the IDSV study evaluated zero-shot.", "b": "Fine-tuning Whisper and MMS on a compiled Indonesian dataset, as the IDSV study did, is within an Indonesian lab's means.", "c": "The IDSV study itself is institution work by ITB and UI with partners, with 6 authors.", "d": "A delivery organisation can fine-tune a released model on client speech, since the IDSV study found the fine-tuned Whisper gave the lowest error." }, "tier_confidence": { "a": "medium", "b": "high", "c": "high", "d": "medium" }, "pipeline_role": "Tier A supplies multilingual base models; Tier C builds Indonesian speech datasets and fine-tunes; Tier B scales them; Tier D brings domain recordings and tests them in proofs of value.", "sources": [ "https://arxiv.org/abs/2410.08828", "https://huggingface.co/indonesian-nlp/wav2vec2-indonesian-javanese-sundanese" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Enhancing Indonesian Automatic Speech Recognition: Evaluating Multilingual Models with Diverse Speech Variabilities", "authors": "Aulia Adila et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2410.08828", "role": "state of the art" }, { "title": "Scaling Speech Technology to 1,000+ Languages", "authors": "Vineel Pratap et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.13516", "role": "state of the art" }, { "title": "NusaWrites: Constructing High-Quality Corpora for Underrepresented and Extremely Low-Resource Languages", "authors": "Samuel Cahyawijaya et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2309.10661", "role": "state of the art" }, { "title": "Robust Speech Recognition via Large-Scale Weak Supervision", "authors": "Alec Radford et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2212.04356", "role": "state of the art" }, { "title": "FLEURS: Few-shot Learning Evaluation of Universal Representations of Speech", "authors": "Alexis Conneau et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2205.12446", "role": "state of the art" } ], "approved": true }, { "id": "streaming-low-latency-speech-recognition", "name": "Streaming and low-latency speech recognition", "definition": "Speech recognition built to transcribe live audio with small delay and small compute, for voice commands, live captions and edge devices.", "compute_anchor": "Moonshine Tiny needs 5 times less compute than Whisper tiny-en for a 10-second segment; Fast Conformer is 2.8 times faster than Conformer and transcribes up to 11 hours.", "data_anchor": "Not in the sources read: the Moonshine and Fast Conformer abstracts do not state training hours.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on Moonshine; 11 on Fast Conformer.", "time_horizon": "Months.", "shelf_life_signals": [ "Streaming models matching offline accuracy at under 200 ms", "On-device recognisers shipped in phone operating systems" ], "exemplars": [ "useful-sensors-moonshine", "nvidia-parakeet-tdt-0-6b-v3", "nvidia-nemotron-speech-streaming-en-0-6b", "mistral-ai-voxtral-mini-4b-realtime-2602" ], "observed_actors": [ "nvidia", "useful-sensors", "mistral-ai", "hugging-face", "systran" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "NVIDIA redesigned Conformer to be 2.8 times faster and scaled it to billions of parameters.", "b": "Small models such as Moonshine Tiny, with 5 times less compute than Whisper tiny-en, fit modest training budgets, though training hours are not in the sources.", "c": "Architecture changes such as Moonshine's padding-free training are paper-scale work with 6 authors.", "d": "Released low-latency checkpoints can run live captioning or voice commands in a proof of concept without training." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A and Tier C publish fast architectures and checkpoints; Tier B adapts them to Indonesian; Tier D puts them into live transcription and voice command pilots.", "sources": [ "https://arxiv.org/abs/2410.15608", "https://arxiv.org/abs/2305.05084" ], "unverified": [ "data_anchor" ], "papers": [ { "title": "Sequence Transduction with Recurrent Neural Networks", "authors": "Alex Graves", "year": 2012, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1211.3711", "role": "founding" }, { "title": "Zipformer: A faster and better encoder for automatic speech recognition", "authors": "Zengwei Yao et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2310.11230", "role": "state of the art" }, { "title": "Fast Conformer with Linearly Scalable Attention for Efficient Speech Recognition", "authors": "Dima Rekesh et al.", "year": 2023, "venue": "ASRU 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2305.05084", "role": "state of the art" }, { "title": "Conformer: Convolution-augmented Transformer for Speech Recognition", "authors": "Anmol Gulati et al.", "year": 2020, "venue": "Interspeech 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2005.08100", "role": "state of the art" }, { "title": "Emformer: Efficient Memory Transformer Based Acoustic Model For Low Latency Streaming Speech Recognition", "authors": "Yangyang Shi et al.", "year": 2020, "venue": "ICASSP 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2010.10759", "role": "state of the art" } ], "approved": true }, { "id": "self-supervised-speech-representations", "name": "Self-supervised speech representation learning", "definition": "Pre-training speech encoders on unlabelled audio so that recognition and other speech tasks need little labelled data.", "compute_anchor": "HuBERT scales to a 1B parameter model; wav2vec 2.0 and WavLM abstracts give no training compute.", "data_anchor": "wav2vec 2.0: 53k hours unlabelled, then 10 minutes labelled gives 4.8/8.2 WER; WavLM: training data scaled from 60k to 94k hours.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on wav2vec 2.0; 19 on WavLM.", "time_horizon": "Months to fine-tune; a year or more to pre-train at 94k hours.", "shelf_life_signals": [ "Supervised or audio language models outperforming self-supervised encoders on low-resource languages" ], "exemplars": [ "meta-wav2vec-2-0", "meta-hubert", "microsoft-wavlm", "meta-w2v-bert-2-0" ], "observed_actors": [ "meta", "microsoft" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Meta and Microsoft pre-trained on 53k to 94k hours and up to 1B parameters.", "b": "Pre-training on tens of thousands of hours of Indonesian audio is a stretch, though fine-tuning the released encoders is not.", "c": "wav2vec 2.0 shows 10 minutes of labelled data can give usable recognition, which suits a unit with small labelled sets.", "d": "Released encoders such as XLSR-53 already underlie an Indonesian, Javanese and Sundanese recogniser, so they can be fine-tuned for a proof of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A pre-trains encoders on unlabelled audio; Tier C and Tier B fine-tune them with small labelled sets; Tier D uses the fine-tuned encoders in speech pilots.", "sources": [ "https://arxiv.org/abs/2006.11477", "https://arxiv.org/abs/2106.07447", "https://arxiv.org/abs/2110.13900" ], "unverified": [], "papers": [ { "title": "wav2vec: Unsupervised Pre-training for Speech Recognition", "authors": "Steffen Schneider et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1904.05862", "role": "founding" }, { "title": "wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations", "authors": "Alexei Baevski et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2006.11477", "role": "founding" }, { "title": "data2vec: A General Framework for Self-supervised Learning in Speech, Vision and Language", "authors": "Alexei Baevski et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2202.03555", "role": "state of the art" }, { "title": "HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units", "authors": "Wei-Ning Hsu et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2106.07447", "role": "state of the art" }, { "title": "WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing", "authors": "Sanyuan Chen et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2110.13900", "role": "state of the art" } ], "approved": true }, { "id": "audio-language-models", "name": "Audio language models for speech understanding", "definition": "Language models that take audio as input to transcribe, answer questions about and converse over speech, sound and music.", "compute_anchor": "Not in the sources read: the Voxtral and Kimi-Audio abstracts give no parameter count or training compute; Kimi-Audio is initialised from a pre-trained LLM.", "data_anchor": "Kimi-Audio: more than 13 million hours of speech, sound and music for pre-training; Voxtral: 32K context handles audio up to 40 minutes.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 106 authors on Voxtral; 40 listed on Kimi-Audio.", "time_horizon": "Months to a year.", "shelf_life_signals": [ "General multimodal models matching dedicated recognisers on transcription accuracy and cost" ], "exemplars": [ "mistral-ai-voxtral-small", "moonshot-ai-kimi-audio", "nvidia-canary-qwen-2-5b", "alibaba-qwen3-asr" ], "observed_actors": [ "mistral-ai", "moonshot-ai", "nvidia", "alibaba", "ibm", "openai", "microsoft" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "not feasible", "tier_d": "feasible", "tier_rationale": { "a": "Mistral and Moonshot built these with 106 and 40 authors and 13 million hours of audio.", "b": "Continued pre-training of an LLM on audio is possible only at far smaller scale than Kimi-Audio's 13 million hours.", "c": "A unit cannot assemble 13 million hours of audio or a team of 40 or more, though it can evaluate released models.", "d": "Open models such as Voxtral under Apache 2.0 can be used directly for long meeting audio up to 40 minutes." }, "tier_confidence": { "a": "high", "b": "low", "c": "medium", "d": "high" }, "pipeline_role": "Tier A trains audio language models; Tier B and Tier C evaluate them on Indonesian speech; Tier D uses them for meeting minutes and call analysis pilots.", "sources": [ "https://arxiv.org/abs/2507.13264", "https://arxiv.org/abs/2504.18425" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "AudioLM: a Language Modeling Approach to Audio Generation", "authors": "Zalán Borsos et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2209.03143", "role": "founding" }, { "title": "Qwen2-Audio Technical Report", "authors": "Yunfei Chu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2407.10759", "role": "state of the art" }, { "title": "Audio Flamingo: A Novel Audio Language Model with Few-Shot Learning and Dialogue Abilities", "authors": "Zhifeng Kong et al.", "year": 2024, "venue": "ICML 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2402.01831", "role": "state of the art" }, { "title": "SALMONN: Towards Generic Hearing Abilities for Large Language Models", "authors": "Changli Tang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2310.13289", "role": "state of the art" }, { "title": "Recent Advances in Speech Language Models: A Survey", "authors": "Wenqian Cui et al.", "year": 2024, "venue": "ACL 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2410.03751", "role": "survey" } ], "approved": true }, { "id": "efficient-speech-encoder-architectures", "name": "Efficient acoustic encoder architectures for speech recognition", "definition": "Designing the convolution and attention blocks of speech recognition encoders to cut compute and memory at equal accuracy.", "compute_anchor": "Conformer reaches 2.7/6.3 WER on LibriSpeech with only 10M parameters; Squeezeformer beats Conformer-CTC at the same FLOPs.", "data_anchor": "LibriSpeech (Conformer, Squeezeformer); LibriSpeech, Aishell-1 and WenetSpeech (Zipformer).", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 11 authors on Conformer; 9 on Zipformer.", "time_horizon": "Months.", "shelf_life_signals": [ "Encoder designs converging so that new blocks give no gain at equal FLOPs" ], "exemplars": [ "google-conformer", "nvidia-fastconformer", "k2-fsa-zipformer", "squeezeformer" ], "observed_actors": [ "google", "nvidia", "k2-fsa", "uc-berkeley", "baidu" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Google and NVIDIA designed Conformer and Fast Conformer with 11-author teams.", "b": "Training on public sets such as LibriSpeech and WenetSpeech is within a national lab's budget.", "c": "A 10M parameter Conformer reaching 2.7/6.3 WER shows the work fits a university compute budget.", "d": "Delivery work uses these encoders inside released models rather than designing new ones." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and Tier C publish encoder designs; Tier B trains them on Indonesian data; Tier D receives them inside released recognisers.", "sources": [ "https://arxiv.org/abs/2005.08100", "https://arxiv.org/abs/2206.00888", "https://arxiv.org/abs/2310.11230" ], "unverified": [], "papers": [ { "title": "Conformer: Convolution-augmented Transformer for Speech Recognition", "authors": "Anmol Gulati et al.", "year": 2020, "venue": "Interspeech 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2005.08100", "role": "founding" }, { "title": "Fast Conformer with Linearly Scalable Attention for Efficient Speech Recognition", "authors": "Dima Rekesh et al.", "year": 2023, "venue": "ASRU 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2305.05084", "role": "state of the art" }, { "title": "Zipformer: A faster and better encoder for automatic speech recognition", "authors": "Zengwei Yao et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2310.11230", "role": "state of the art" }, { "title": "Squeezeformer: An Efficient Transformer for Automatic Speech Recognition", "authors": "Sehoon Kim et al.", "year": 2022, "venue": "NeurIPS 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2206.00888", "role": "state of the art" }, { "title": "Branchformer: Parallel MLP-Attention Architectures to Capture Local and Global Context for Speech Recognition and Understanding", "authors": "Yifan Peng et al.", "year": 2022, "venue": "ICML 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2207.02971", "role": "state of the art" } ], "approved": true }, { "id": "zero-shot-voice-cloning-tts", "name": "Large-scale zero-shot text-to-speech", "definition": "Text-to-speech trained on tens of thousands of hours that speaks in an unseen voice from a few seconds of reference audio.", "compute_anchor": "F5-TTS reaches an inference real-time factor of 0.15; VALL-E, F5-TTS and CosyVoice abstracts give no training compute.", "data_anchor": "VALL-E: 60K hours of English speech and a 3-second prompt; F5-TTS: a public 100K hours multilingual dataset.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 13 authors on VALL-E; 8 on F5-TTS.", "time_horizon": "Months.", "shelf_life_signals": [ "Voice cloning restricted by law or platform policy", "Open models matching commercial arena leaders" ], "exemplars": [ "microsoft-vall-e", "swivid-f5-tts", "funaudiollm-cosyvoice-2", "coqui-xtts-v2" ], "observed_actors": [ "microsoft", "funaudiollm", "swivid", "elevenlabs", "minimax", "fish-audio", "amphion", "coqui", "resemble-ai", "zyphra" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Microsoft scaled TTS training to 60K hours for VALL-E.", "b": "F5-TTS shows a public 100K hours dataset is enough, which an Indonesian lab can combine with local speech.", "c": "Collecting tens of thousands of hours is a stretch for a unit, though fine-tuning released models such as F5-TTS is not.", "d": "Released models clone a voice from 3 seconds of audio, enough for a proof of concept with consented voices." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A and open labs release zero-shot TTS; Tier B adds Indonesian data; Tier C evaluates naturalness; Tier D builds voice products with consented voices.", "sources": [ "https://arxiv.org/abs/2301.02111", "https://arxiv.org/abs/2410.06885", "https://arxiv.org/abs/2407.05407" ], "unverified": [], "papers": [ { "title": "Neural Codec Language Models are Zero-Shot Text to Speech Synthesizers", "authors": "Chengyi Wang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2301.02111", "role": "founding" }, { "title": "NaturalSpeech 3: Zero-Shot Speech Synthesis with Factorized Codec and Diffusion Models", "authors": "Zeqian Ju et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.03100", "role": "state of the art" }, { "title": "XTTS: a Massively Multilingual Zero-Shot Text-to-Speech Model", "authors": "Edresson Casanova et al.", "year": 2024, "venue": "INTERSPEECH 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2406.04904", "role": "state of the art" }, { "title": "F5-TTS: A Fairytaler that Fakes Fluent and Faithful Speech with Flow Matching", "authors": "Yushen Chen et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2410.06885", "role": "state of the art" }, { "title": "CosyVoice: A Scalable Multilingual Zero-shot Text-to-speech Synthesizer based on Supervised Semantic Tokens", "authors": "Zhihao Du et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2407.05407", "role": "state of the art" } ], "approved": true }, { "id": "neural-vocoders", "name": "Neural vocoders", "definition": "Networks that turn spectrograms or acoustic features into audio waveforms quickly and with high fidelity.", "compute_anchor": "HiFi-GAN generates 22.05 kHz audio 167.9 times faster than real time on one V100 and 13.4 times on CPU; BigVGAN trains a GAN vocoder of up to 112M parameters.", "data_anchor": "BigVGAN is trained only on clean speech (LibriTTS) and generalises to unseen speakers, languages and music.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on HiFi-GAN; 5 on BigVGAN.", "time_horizon": "Months.", "shelf_life_signals": [ "Codec-token and flow-matching TTS that decode waveforms without a separate vocoder" ], "exemplars": [ "kakao-enterprise-hifi-gan", "nvidia-bigvgan", "google-deepmind-wavenet" ], "observed_actors": [ "kakao-enterprise", "nvidia", "google-deepmind", "descript", "naver", "baidu" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "NVIDIA trained BigVGAN at up to 112M parameters.", "b": "HiFi-GAN runs 167.9 times faster than real time on one V100, so training and serving fit a national lab.", "c": "HiFi-GAN came from a 3-author team and trains on public speech, which suits a unit.", "d": "The small HiFi-GAN runs 13.4 times faster than real time on CPU, so it can ship in an on-premise proof of concept." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and Tier C publish vocoders; Tier B pairs them with Indonesian acoustic models; Tier D deploys them inside TTS pipelines.", "sources": [ "https://arxiv.org/abs/2010.05646", "https://arxiv.org/abs/2206.04658", "https://arxiv.org/abs/1609.03499" ], "unverified": [], "papers": [ { "title": "WaveNet: A Generative Model for Raw Audio", "authors": "Aaron van den Oord et al.", "year": 2016, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1609.03499", "role": "founding" }, { "title": "Vocos: Closing the gap between time-domain and Fourier-based neural vocoders for high-quality audio synthesis", "authors": "Hubert Siuzdak", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.00814", "role": "state of the art" }, { "title": "BigVGAN: A Universal Neural Vocoder with Large-Scale Training", "authors": "Sang-gil Lee et al.", "year": 2022, "venue": "ICLR 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2206.04658", "role": "state of the art" }, { "title": "HiFi-GAN: Generative Adversarial Networks for Efficient and High Fidelity Speech Synthesis", "authors": "Jungil Kong et al.", "year": 2020, "venue": "NeurIPS 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2010.05646", "role": "state of the art" }, { "title": "WaveGlow: A Flow-based Generative Network for Speech Synthesis", "authors": "Ryan Prenger et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1811.00002", "role": "state of the art" } ], "approved": true }, { "id": "real-time-voice-interaction", "name": "Real-time spoken dialogue and low-latency voice generation", "definition": "Models that listen and speak with delays of a few hundred milliseconds, including full-duplex speech-to-speech dialogue and streaming speech generation for voice agents.", "compute_anchor": "Moshi: theoretical latency 160 ms, 200 ms in practice; parameter count and training compute are not in the abstract.", "data_anchor": "Not in the sources read: the Moshi abstract does not state training hours.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 8 authors on Moshi.", "time_horizon": "Months to a year.", "shelf_life_signals": [ "Voice agents reaching human turn-taking latency in Indonesian", "Cascaded ASR, LLM and TTS pipelines matching speech-to-speech latency" ], "exemplars": [ "kyutai-moshiko", "openai-gpt-realtime-2", "google-deepmind-gemini-3-8-live", "sesame-csm-1b" ], "observed_actors": [ "kyutai", "openai", "google-deepmind", "amazon", "sesame", "cartesia", "elevenlabs", "bytedance" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Kyutai built the first real-time full-duplex spoken LLM at 200 ms practical latency with an 8-author team.", "b": "Training a speech-text foundation model like Moshi needs data and compute not stated in the source, so the grade is uncertain.", "c": "Units can study turn-taking and latency with open models such as Moshi but are unlikely to train one.", "d": "API voice models with latency near 200 ms support voice-bot and contact-centre proofs of value." }, "tier_confidence": { "a": "high", "b": "low", "c": "low", "d": "high" }, "pipeline_role": "Tier A serves real-time voice models; Tier B and Tier C test them on Indonesian conversations; Tier D builds voice agents on them for clients.", "sources": [ "https://arxiv.org/abs/2410.00037" ], "unverified": [ "data_anchor" ], "papers": [ { "title": "Generative Spoken Dialogue Language Modeling", "authors": "Tu Anh Nguyen et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2203.16502", "role": "founding" }, { "title": "Qwen2.5-Omni Technical Report", "authors": "Jin Xu et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2503.20215", "role": "state of the art" }, { "title": "Mini-Omni: Language Models Can Hear, Talk While Thinking in Streaming", "authors": "Zhifei Xie et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2408.16725", "role": "state of the art" }, { "title": "LLaMA-Omni: Seamless Speech Interaction with Large Language Models", "authors": "Qingkai Fang et al.", "year": 2024, "venue": "ICLR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2409.06666", "role": "state of the art" }, { "title": "Moshi: a speech-text foundation model for real-time dialogue", "authors": "Alexandre Défossez et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2410.00037", "role": "state of the art" } ], "approved": true }, { "id": "on-device-text-to-speech", "name": "On-device text-to-speech", "definition": "Small text-to-speech models that run offline on phones, CPUs or microcontrollers without a cloud service.", "compute_anchor": "Kokoro: 82 million parameters, about 1,000 A100 80GB GPU hours in total (about USD 1,000); sanoTTS: 294k to 2.27M parameters, real time on an ESP32-S3.", "data_anchor": "Kokoro card: under 100 hours of training data for v0.19 and a few hundred hours for v1.0; sanoTTS covers 11 voices across 6 languages including Indonesian.", "team_size_proxy": "Proxy only: not in the sources read (model cards, no papers).", "time_horizon": "Weeks to months.", "shelf_life_signals": [ "Operating systems shipping good offline voices in Indonesian" ], "exemplars": [ "hexgrad-kokoro-82m", "ampixa-sanotts", "rhasspy-piper", "neuphonic-neutts-air" ], "observed_actors": [ "hexgrad", "ampixa", "rhasspy", "neuphonic", "silero", "myshell" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs could do this easily, though the exemplars come from small teams.", "b": "Kokoro was trained for about USD 1,000 of A100 time, well within a national lab budget.", "c": "An 82M parameter model trained on about 1,000 GPU hours fits a university unit.", "d": "sanoTTS already speaks Indonesian on a USD 3 microcontroller, so offline voice proofs of concept need no training." }, "tier_confidence": { "a": "medium", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Small teams release compact voices; Tier C and Tier B train Indonesian and regional voices; Tier D puts them on devices for offline field use.", "sources": [ "https://huggingface.co/hexgrad/Kokoro-82M", "https://huggingface.co/ampixa/sanoTTS" ], "unverified": [ "team_size_proxy" ], "papers": [ { "title": "Natural TTS Synthesis by Conditioning WaveNet on Mel Spectrogram Predictions", "authors": "Jonathan Shen et al.", "year": 2017, "venue": "ICASSP 2018 (per arXiv comment)", "url": "https://arxiv.org/abs/1712.05884", "role": "founding" }, { "title": "Matcha-TTS: A fast TTS architecture with conditional flow matching", "authors": "Shivam Mehta et al.", "year": 2023, "venue": "ICASSP 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2309.03199", "role": "state of the art" }, { "title": "StyleTTS 2: Towards Human-Level Text-to-Speech through Style Diffusion and Adversarial Training with Large Speech Language Models", "authors": "Yinghao Aaron Li et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2306.07691", "role": "state of the art" }, { "title": "Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech", "authors": "Jaehyeon Kim et al.", "year": 2021, "venue": "ICML 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2106.06103", "role": "state of the art" }, { "title": "LightSpeech: Lightweight and Fast Text to Speech with Neural Architecture Search", "authors": "Renqian Luo et al.", "year": 2021, "venue": "ICASSP 21 (per arXiv comment)", "url": "https://arxiv.org/abs/2102.04040", "role": "state of the art" } ], "approved": true }, { "id": "tts-low-resource-languages", "name": "Text-to-speech for low-resource languages", "definition": "Building speech synthesis for languages with little recorded speech, from massively multilingual systems to regional voices.", "compute_anchor": "Not in the sources read: MMS and Indic Parler-TTS give no training compute.", "data_anchor": "MMS: speech synthesis models for 1,107 languages from readings of religious texts; Indic Parler-TTS: 1,806 hours of Indic and English speech for 20 Indic languages and English.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 16 authors on MMS.", "time_horizon": "Months.", "shelf_life_signals": [ "Commercial TTS covering Indonesian regional languages such as Javanese and Sundanese at high quality" ], "exemplars": [ "ai4bharat-indic-parler-tts", "meta-mms" ], "observed_actors": [ "meta", "ai4bharat", "elevenlabs", "ampixa" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Meta built synthesis for 1,107 languages in MMS with a 16-author team.", "b": "AI4Bharat shows a regional lab can train a 20-language TTS on 1,806 hours, a model for Indonesian and regional languages.", "c": "Recording and curating regional speech is institution-scale work, as the Indic Parler-TTS data shows.", "d": "Delivery teams can use existing Indonesian voices but rarely have the data to build new regional languages." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A releases multilingual synthesis; Tier C and Tier B record and train Indonesian regional voices; Tier D uses them in public service and education pilots.", "sources": [ "https://arxiv.org/abs/2305.13516", "https://huggingface.co/ai4bharat/indic-parler-tts" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Learning to Speak Fluently in a Foreign Language: Multilingual Speech Synthesis and Cross-Language Voice Cloning", "authors": "Yu Zhang et al.", "year": 2019, "venue": "Interspeech 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1907.04448", "role": "founding" }, { "title": "XTTS: a Massively Multilingual Zero-Shot Text-to-Speech Model", "authors": "Edresson Casanova et al.", "year": 2024, "venue": "INTERSPEECH 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2406.04904", "role": "state of the art" }, { "title": "Scaling Speech Technology to 1,000+ Languages", "authors": "Vineel Pratap et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.13516", "role": "state of the art" }, { "title": "YourTTS: Towards Zero-Shot Multi-Speaker TTS and Zero-Shot Voice Conversion for everyone", "authors": "Edresson Casanova et al.", "year": 2021, "venue": "Proceedings of the 39th International Conference on Machine Learning, PMLR 162:2", "url": "https://arxiv.org/abs/2112.02418", "role": "state of the art" } ], "approved": true }, { "id": "expressive-controllable-speech-synthesis", "name": "Expressive and controllable speech synthesis", "definition": "Controlling the style, emotion and identity of synthetic speech from a text description or a sampled style rather than only a reference recording.", "compute_anchor": "Not in the sources read: StyleTTS 2 and the Parler-TTS paper abstracts give no training compute.", "data_anchor": "Parler-TTS paper: automatic labels of speaker identity and style applied to a 45k hour dataset; StyleTTS 2: LJSpeech, VCTK and LibriTTS.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 2 authors on the natural-language guidance paper; 5 on StyleTTS 2.", "time_horizon": "Months.", "shelf_life_signals": [ "General TTS models accepting free-text style prompts as standard" ], "exemplars": [ "columbia-university-styletts-2", "maya-research-maya1", "amphion-vevo2" ], "observed_actors": [ "columbia-university", "maya-research", "amphion", "nvidia", "nari-labs", "stability-ai" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs can label and train at the 45k hour scale used for natural-language guidance.", "b": "The 45k hour labelling method uses found data, which an Indonesian lab can apply to local speech.", "c": "StyleTTS 2 reached human-level naturalness on public sets with a 5-author team.", "d": "Released description-controlled models can voice branded or emotional content in a proof of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and Tier C publish control methods; Tier B adapts them to Indonesian prosody; Tier D applies them to content and customer voice pilots.", "sources": [ "https://arxiv.org/abs/2306.07691", "https://arxiv.org/abs/2402.01912" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Style Tokens: Unsupervised Style Modeling, Control and Transfer in End-to-End Speech Synthesis", "authors": "Yuxuan Wang et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1803.09017", "role": "founding" }, { "title": "Towards End-to-End Prosody Transfer for Expressive Speech Synthesis with Tacotron", "authors": "RJ Skerry-Ryan et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1803.09047", "role": "founding" }, { "title": "Natural language guidance of high-fidelity text-to-speech with synthetic annotations", "authors": "Dan Lyth et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.01912", "role": "state of the art" }, { "title": "NaturalSpeech 2: Latent Diffusion Models are Natural and Zero-Shot Speech and Singing Synthesizers", "authors": "Kai Shen et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2304.09116", "role": "state of the art" }, { "title": "StyleTTS 2: Towards Human-Level Text-to-Speech through Style Diffusion and Adversarial Training with Large Speech Language Models", "authors": "Yinghao Aaron Li et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2306.07691", "role": "state of the art" } ], "approved": true }, { "id": "text-to-music-generation", "name": "Text-to-music and song generation", "definition": "Generating music, including vocals that follow given lyrics, from text descriptions, melodies or reference audio.", "compute_anchor": "ACE-Step generates up to 4 minutes of music in 20 seconds on one A100; training compute is not in the MusicGen, YuE or ACE-Step abstracts.", "data_anchor": "YuE scales to trillions of tokens and generates up to five minutes of music; MusicGen trains a single-stage LM over compressed music tokens.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 8 authors on MusicGen; 58 on YuE; 5 on ACE-Step.", "time_horizon": "Months.", "shelf_life_signals": [ "Licensing settlements limiting training to licensed music", "Open models matching Suno on arena boards" ], "exemplars": [ "meta-musicgen", "m-a-p-yue-1", "stepfun-ace-step-1-5", "suno-v6" ], "observed_actors": [ "suno", "uncharted-labs", "google-deepmind", "meta", "m-a-p", "stepfun", "ace-studio", "stability-ai", "minimax", "elevenlabs" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Google DeepMind, Suno and Meta lead, and YuE needed 58 authors and trillions of tokens.", "b": "Licensed music data at YuE's scale is hard to obtain, though fine-tuning open models such as ACE-Step is possible.", "c": "Units can fine-tune open models that run 4 minutes of music in 20 seconds on one A100, but not train at scale.", "d": "Open and API music models can produce licensed background music for content proofs of concept." }, "tier_confidence": { "a": "high", "b": "low", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A trains music models; Tier C studies control and evaluation; Tier B adds Indonesian styles; Tier D uses licensed outputs in media pilots.", "sources": [ "https://arxiv.org/abs/2306.05284", "https://arxiv.org/abs/2503.08638", "https://arxiv.org/abs/2506.00045" ], "unverified": [], "papers": [ { "title": "Jukebox: A Generative Model for Music", "authors": "Prafulla Dhariwal et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2005.00341", "role": "founding" }, { "title": "YuE: Scaling Open Foundation Models for Long-Form Music Generation", "authors": "Ruibin Yuan et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2503.08638", "role": "state of the art" }, { "title": "Fast Timing-Conditioned Latent Audio Diffusion", "authors": "Zach Evans et al.", "year": 2024, "venue": "ICML 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2402.04825", "role": "state of the art" }, { "title": "MusicLM: Generating Music From Text", "authors": "Andrea Agostinelli et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2301.11325", "role": "state of the art" }, { "title": "Foundation Models for Music: A Survey", "authors": "Yinghao Ma et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2408.14340", "role": "survey" } ], "approved": true }, { "id": "video-to-audio-foley-generation", "name": "Video-to-audio and Foley generation", "definition": "Generating sound effects and audio that match what happens in a video, in time and in meaning.", "compute_anchor": "MMAudio has 157M parameters and generates an 8-second clip in 1.23 seconds.", "data_anchor": "HunyuanVideo-Foley curates a 100k-hour multimodal dataset by automated annotation; MMAudio trains jointly on video and text-audio data.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on MMAudio; 8 on HunyuanVideo-Foley.", "time_horizon": "Months.", "shelf_life_signals": [ "Video generators producing synchronised audio natively" ], "exemplars": [ "mmaudio", "tencent-hunyuanvideo-foley", "funaudiollm-thinksound" ], "observed_actors": [ "uiuc", "sony", "tencent", "funaudiollm", "shanghai-ai-lab" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Tencent built a 100k-hour data pipeline for HunyuanVideo-Foley.", "b": "A 157M parameter model such as MMAudio is small enough for a national lab to retrain.", "c": "MMAudio came from a 6-author academic team at 157M parameters.", "d": "MMAudio generates 8 seconds in 1.23 seconds, fast enough for content production proofs of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and Tier C release video-to-audio models; Tier B adds local sound libraries; Tier D applies them in media workflows.", "sources": [ "https://arxiv.org/abs/2412.15322", "https://arxiv.org/abs/2508.16930" ], "unverified": [], "papers": [ { "title": "Visually Indicated Sounds", "authors": "Andrew Owens et al.", "year": 2015, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1512.08512", "role": "founding" }, { "title": "FoleyCrafter: Bring Silent Videos to Life with Lifelike and Synchronized Sounds", "authors": "Yiming Zhang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2407.01494", "role": "state of the art" }, { "title": "Frieren: Efficient Video-to-Audio Generation Network with Rectified Flow Matching", "authors": "Yongqi Wang et al.", "year": 2024, "venue": "NeurIPS 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2406.00320", "role": "state of the art" }, { "title": "MMAudio: Taming Multimodal Joint Training for High-Quality Video-to-Audio Synthesis", "authors": "Ho Kei Cheng et al.", "year": 2024, "venue": "CVPR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2412.15322", "role": "state of the art" }, { "title": "Diff-Foley: Synchronized Video-to-Audio Synthesis with Latent Diffusion Models", "authors": "Simian Luo et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.17203", "role": "state of the art" } ], "approved": true }, { "id": "neural-audio-codecs", "name": "Neural audio codecs and audio tokenisers", "definition": "Learned encoders and decoders that compress audio into discrete tokens, used for transmission and as the vocabulary of speech and music generators.", "compute_anchor": "SoundStream runs in real time on a smartphone CPU and covers 3 to 18 kbps in one model.", "data_anchor": "Not in the sources read: the SoundStream abstract does not state training data size.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on SoundStream.", "time_horizon": "Months.", "shelf_life_signals": [ "Generators that work on continuous latents instead of discrete codec tokens" ], "exemplars": [ "google-soundstream", "kyutai-mimi" ], "observed_actors": [ "google", "kyutai", "neuphonic", "heartmula" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Google and Kyutai built the codecs behind their speech and music generators.", "b": "A codec that runs in real time on a smartphone CPU is small enough to train at a national lab.", "c": "SoundStream came from a 5-author team, a size a unit can match.", "d": "Delivery work uses codecs inside released generators rather than building them." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A and Tier C build codecs; Tier B uses them as tokenisers for Indonesian speech models; Tier D receives them inside TTS and dialogue systems.", "sources": [ "https://arxiv.org/abs/2107.03312", "https://arxiv.org/abs/2410.00037" ], "unverified": [ "data_anchor" ], "papers": [ { "title": "SoundStream: An End-to-End Neural Audio Codec", "authors": "Neil Zeghidour et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2107.03312", "role": "founding" }, { "title": "WavTokenizer: an Efficient Acoustic Discrete Codec Tokenizer for Audio Language Modeling", "authors": "Shengpeng Ji et al.", "year": 2024, "venue": "ICLR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2408.16532", "role": "state of the art" }, { "title": "High-Fidelity Audio Compression with Improved RVQGAN", "authors": "Rithesh Kumar et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2306.06546", "role": "state of the art" }, { "title": "SpeechTokenizer: Unified Speech Tokenizer for Speech Large Language Models", "authors": "Xin Zhang et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2308.16692", "role": "state of the art" }, { "title": "Codec-SUPERB: An In-Depth Analysis of Sound Codec Models", "authors": "Haibin Wu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.13071", "role": "survey" } ], "approved": true }, { "id": "speech-enhancement-source-separation", "name": "Speech enhancement and audio source separation", "definition": "Removing noise from speech and splitting mixtures into separate speakers, instruments or sounds.", "compute_anchor": "SepFormer stays competitive when downsampling its encoded representation by a factor of 8; training compute is not in the abstracts.", "data_anchor": "SepFormer: WSJ0-2mix and WSJ0-3mix (SI-SNRi 22.3 dB and 19.5 dB); MossFormer2: WSJ0-2/3mix, Libri2Mix and WHAM!/WHAMR!.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on SepFormer; 10 on MossFormer2.", "time_horizon": "Months.", "shelf_life_signals": [ "Recognisers robust enough to noise that separate enhancement adds nothing" ], "exemplars": [ "speechbrain-sepformer", "alibaba-mossformer2", "meta-sam-audio" ], "observed_actors": [ "speechbrain", "alibaba", "meta", "inria" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Meta and Alibaba release separation and enhancement models.", "b": "Public mixtures such as WSJ0-2mix and Libri2Mix make training affordable for a national lab.", "c": "SepFormer came from a 5-author team on standard public benchmarks.", "d": "Released enhancement models can clean call-centre and field recordings before transcription in a pilot." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and Tier C publish separation models; Tier B tunes them for local noise; Tier D puts them in front of transcription in pilots.", "sources": [ "https://arxiv.org/abs/2010.13154", "https://arxiv.org/abs/2312.11825" ], "unverified": [], "papers": [ { "title": "Conv-TasNet: Surpassing Ideal Time-Frequency Magnitude Masking for Speech Separation", "authors": "Yi Luo et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1809.07454", "role": "founding" }, { "title": "Music Source Separation with Band-Split RoPE Transformer", "authors": "Wei-Tsung Lu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2309.02612", "role": "state of the art" }, { "title": "Hybrid Transformers for Music Source Separation", "authors": "Simon Rouard et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2211.08553", "role": "state of the art" }, { "title": "Attention is All You Need in Speech Separation", "authors": "Cem Subakan et al.", "year": 2020, "venue": "ICASSP 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2010.13154", "role": "state of the art" }, { "title": "DCCRN: Deep Complex Convolution Recurrent Network for Phase-Aware Speech Enhancement", "authors": "Yanxin Hu et al.", "year": 2020, "venue": "Interspeech 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2008.00264", "role": "state of the art" } ], "approved": true }, { "id": "general-audio-music-representations", "name": "General audio and music representation learning", "definition": "Pre-training audio encoders that classify and describe sound events, music and speech, often without labels.", "compute_anchor": "Dasheng: 1.2 billion parameters; MERT scales from 95M to 330M parameters.", "data_anchor": "Dasheng: 272,356 hours of diverse audio; AST: 0.485 mAP on AudioSet.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on AST; 20 on MERT; 6 on Dasheng.", "time_horizon": "Months.", "shelf_life_signals": [ "Audio language models replacing dedicated classifiers" ], "exemplars": [ "mit-ast", "xiaomi-dasheng", "m-a-p-mert", "google-yamnet" ], "observed_actors": [ "mit", "xiaomi", "m-a-p", "google", "tencent", "universitat-pompeu-fabra" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Xiaomi trained Dasheng at 1.2 billion parameters on 272,356 hours.", "b": "MERT at 95M to 330M parameters shows useful encoders are within a national lab's reach.", "c": "AST came from a 3-author team and reached 0.485 mAP on AudioSet.", "d": "Released classifiers such as YAMNet can tag sounds in an acoustic monitoring proof of concept without training." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A and Tier C release encoders; Tier B tunes them for local sounds and music; Tier D uses them for machine sound and media tagging pilots.", "sources": [ "https://arxiv.org/abs/2104.01778", "https://arxiv.org/abs/2406.06992", "https://arxiv.org/abs/2306.00107" ], "unverified": [], "papers": [ { "title": "PANNs: Large-Scale Pretrained Audio Neural Networks for Audio Pattern Recognition", "authors": "Qiuqiang Kong et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1912.10211", "role": "founding" }, { "title": "MERT: Acoustic Music Understanding Model with Large-Scale Self-supervised Training", "authors": "Yizhi Li et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2306.00107", "role": "state of the art" }, { "title": "CLAP: Learning Audio Concepts From Natural Language Supervision", "authors": "Benjamin Elizalde et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2206.04769", "role": "state of the art" }, { "title": "Masked Autoencoders that Listen", "authors": "Po-Yao Huang et al.", "year": 2022, "venue": "NeurIPS 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2207.06405", "role": "state of the art" }, { "title": "BEATs: Audio Pre-Training with Acoustic Tokenizers", "authors": "Sanyuan Chen et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2212.09058", "role": "state of the art" } ], "approved": true }, { "id": "bioacoustic-monitoring", "name": "Bioacoustic classification for biodiversity monitoring", "definition": "Recognising birds, marine life and other animals in passive acoustic recordings to monitor biodiversity.", "compute_anchor": "Not in the sources read: the Perch and SurfPerch abstracts give no training compute.", "data_anchor": "Perch: bird vocalisation embeddings transfer to bat, marine mammal and amphibian calls; SurfPerch: ReefSet reef library at 2% of the sample count of bird libraries.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on the Perch transfer study; 15 on SurfPerch.", "time_horizon": "Months.", "shelf_life_signals": [ "Embeddings that classify most taxa few-shot, leaving only data collection" ], "exemplars": [ "google-perch", "google-surfperch", "birdnet-team-birdnet" ], "observed_actors": [ "google", "birdnet-team" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google builds the Perch and SurfPerch embedding models.", "b": "Few-shot transfer from bird embeddings, as Perch shows, needs only small labelled sets of local species.", "c": "SurfPerch shows a reef library at 2% of bird sample counts suffices, so a unit can build one for Indonesian reefs.", "d": "Released classifiers such as BirdNET process large survey audio without training, fitting environmental monitoring pilots." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A releases embedding models; Tier C records and labels local species; Tier B scales them nationally; Tier D runs monitoring for plantations and conservation clients.", "sources": [ "https://arxiv.org/abs/2307.06292", "https://arxiv.org/abs/2404.16436", "https://github.com/kahst/BirdNET-Analyzer" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Global birdsong embeddings enable superior transfer learning for bioacoustic classification", "authors": "Burooj Ghani et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2307.06292", "role": "state of the art" }, { "title": "BEANS: The Benchmark of Animal Sounds", "authors": "Masato Hagiwara et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2210.12300", "role": "state of the art" }, { "title": "AVES: Animal Vocalization Encoder based on Self-Supervision", "authors": "Masato Hagiwara", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2210.14493", "role": "state of the art" }, { "title": "Computational bioacoustics with deep learning: a review and roadmap", "authors": "Dan Stowell", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2112.06725", "role": "survey" } ], "approved": true }, { "id": "voice-activity-turn-detection", "name": "Voice activity, turn and keyword detection", "definition": "Small models that detect when someone speaks, when they finish a turn, or which short command they said, as the front end of speech systems.", "compute_anchor": "MarbleNet matches a state-of-the-art VAD with about one tenth of the parameters; Smart Turn v3 has 8M parameters and its card announces CPU inference in 12 ms.", "data_anchor": "MatchboxNet: Google Speech Commands with an auxiliary noise dataset for augmentation.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on MarbleNet; 2 on MatchboxNet.", "time_horizon": "Weeks to months.", "shelf_life_signals": [ "Full-duplex dialogue models that handle turn-taking internally" ], "exemplars": [ "nvidia-marblenet", "pipecat-smart-turn-v3", "alibaba-fsmn-vad" ], "observed_actors": [ "nvidia", "pipecat", "alibaba", "rednote", "speechbrain" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "NVIDIA publishes MarbleNet and MatchboxNet in NeMo.", "b": "Models with a tenth of the usual parameters train cheaply on Indonesian audio.", "c": "MarbleNet and MatchboxNet came from 2 to 3 author teams on public data.", "d": "An 8M parameter turn detector running on CPU drops straight into a voice-bot proof of concept." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and Tier C release small detectors; Tier B tunes them for Indonesian; Tier D uses them as the front end of voice agents.", "sources": [ "https://arxiv.org/abs/2010.13886", "https://arxiv.org/abs/2004.08531", "https://huggingface.co/pipecat-ai/smart-turn-v3" ], "unverified": [], "papers": [ { "title": "TurnGPT: a Transformer-based Language Model for Predicting Turn-taking in Spoken Dialog", "authors": "Erik Ekstedt et al.", "year": 2020, "venue": "EMNLP 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2010.10874", "role": "founding" }, { "title": "Turn-taking and Backchannel Prediction with Acoustic and Large Language Model Fusion", "authors": "Jinhan Wang et al.", "year": 2024, "venue": "ICASSP 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2401.14717", "role": "state of the art" }, { "title": "Voice Activity Projection: Self-supervised Learning of Turn-taking Events", "authors": "Erik Ekstedt et al.", "year": 2022, "venue": "INTERSPEECH 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2205.09812", "role": "state of the art" }, { "title": "MarbleNet: Deep 1D Time-Channel Separable Convolutional Neural Network for Voice Activity Detection", "authors": "Fei Jia et al.", "year": 2020, "venue": "ICASSP 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2010.13886", "role": "state of the art" } ], "approved": true }, { "id": "speech-emotion-recognition", "name": "Speech emotion recognition", "definition": "Recognising emotion, arousal and valence from the sound of speech across languages and recording conditions.", "compute_anchor": "Not in the sources read: the emotion2vec abstract gives no compute; only linear layers are trained for the downstream task.", "data_anchor": "emotion2vec: pre-trained on open-source unlabelled emotion data, tested on IEMOCAP and 10 languages of emotion datasets.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 7 authors on emotion2vec.", "time_horizon": "Months.", "shelf_life_signals": [ "Audio language models reporting emotion as a standard output", "Regulation limiting emotion recognition in workplaces" ], "exemplars": [ "alibaba-emotion2vec-plus-large", "audeering-wav2vec2-emotion-msp-dim" ], "observed_actors": [ "alibaba", "audeering", "speechbrain" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Alibaba released emotion2vec as a universal emotion representation.", "b": "Training only linear layers on emotion2vec, as the paper does, makes Indonesian adaptation cheap.", "c": "A 7-author team built emotion2vec on open-source data.", "d": "Released models can flag caller emotion in contact-centre proofs of concept, subject to privacy rules." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and Tier C release emotion encoders; Tier B labels Indonesian emotional speech; Tier D applies them in call analytics pilots.", "sources": [ "https://arxiv.org/abs/2312.15185" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "SUPERB: Speech processing Universal PERformance Benchmark", "authors": "Shu-wen Yang et al.", "year": 2021, "venue": "Interspeech 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2105.01051", "role": "founding" }, { "title": "EmoBox: Multilingual Multi-corpus Speech Emotion Recognition Toolkit and Benchmark", "authors": "Ziyang Ma et al.", "year": 2024, "venue": "INTERSPEECH 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2406.07162", "role": "state of the art" }, { "title": "emotion2vec: Self-Supervised Pre-Training for Speech Emotion Representation", "authors": "Ziyang Ma et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.15185", "role": "state of the art" }, { "title": "Emotion Recognition from Speech Using Wav2vec 2.0 Embeddings", "authors": "Leonardo Pepino et al.", "year": 2021, "venue": "Interspeech 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2104.03502", "role": "state of the art" } ], "approved": true }, { "id": "speaker-diarization", "name": "Speaker diarization", "definition": "Working out who spoke when in a recording with several speakers, including overlapping speech.", "compute_anchor": "Not in the sources read: the powerset and Sortformer abstracts give no compute.", "data_anchor": "Powerset loss: 9 diarization benchmarks; TitaNet: DER 1.73% on AMI-MixHeadset.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 2 authors on the powerset paper; 9 on Sortformer.", "time_horizon": "Months.", "shelf_life_signals": [ "Speech-to-text models that output speaker labels natively, as Sortformer aims for" ], "exemplars": [ "pyannoteai-speaker-diarization-3-1", "nvidia-sortformer", "brno-university-of-technology-diarizen" ], "observed_actors": [ "pyannoteai", "nvidia", "brno-university-of-technology", "openmoss" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "NVIDIA built Sortformer to put speaker labels inside speech-to-text models.", "b": "Public benchmarks such as the 9 used for the powerset loss allow national labs to train and test.", "c": "The powerset loss came from a 2-author team, so diarization research fits a unit.", "d": "Open pipelines such as pyannote 3.1 can split meeting recordings by speaker in a minutes-taking pilot." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and Tier C release diarization models; Tier B tests them on Indonesian meetings; Tier D builds meeting minutes and call analytics with them.", "sources": [ "https://arxiv.org/abs/2310.13025", "https://arxiv.org/abs/2409.06656", "https://arxiv.org/abs/2110.04410" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Fully Supervised Speaker Diarization", "authors": "Aonan Zhang et al.", "year": 2018, "venue": "ICASSP 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1810.04719", "role": "founding" }, { "title": "Sortformer: A Novel Approach for Permutation-Resolved Speaker Supervision in Speech-to-Text Systems", "authors": "Taejin Park et al.", "year": 2024, "venue": "Proceedings of the 42nd International Conference on Machine Learning (ICML), 202", "url": "https://arxiv.org/abs/2409.06656", "role": "state of the art" }, { "title": "Powerset multi-class cross entropy loss for neural speaker diarization", "authors": "Alexis Plaquet et al.", "year": 2023, "venue": "INTERSPEECH 2023, Aug 2023, Dublin, Ireland. pp.3222-3226", "url": "https://arxiv.org/abs/2310.13025", "role": "state of the art" }, { "title": "End-to-End Neural Speaker Diarization with Self-attention", "authors": "Yusuke Fujita et al.", "year": 2019, "venue": "ASRU 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1909.06247", "role": "state of the art" }, { "title": "A Review of Speaker Diarization: Recent Advances with Deep Learning", "authors": "Tae Jin Park et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2101.09624", "role": "survey" } ], "approved": true }, { "id": "speaker-verification-embeddings", "name": "Speaker verification embeddings", "definition": "Mapping a voice to a fixed-length embedding to verify or identify the speaker.", "compute_anchor": "TitaNet-S has only 6M parameters; CAM++ targets ECAPA-TDNN accuracy at lower computational cost.", "data_anchor": "VoxCeleb (ECAPA-TDNN, TitaNet EER 0.68%) and CN-Celeb (CAM++).", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on ECAPA-TDNN; 5 on CAM++.", "time_horizon": "Months.", "shelf_life_signals": [ "Voice cloning defeating voice biometrics, shifting work to spoofing detection" ], "exemplars": [ "speechbrain-ecapa-tdnn", "nvidia-titanet", "alibaba-cam-plus-plus" ], "observed_actors": [ "speechbrain", "nvidia", "alibaba" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "NVIDIA and Alibaba publish speaker embedding models.", "b": "Models of 6M parameters trained on public VoxCeleb data are cheap for a national lab.", "c": "ECAPA-TDNN came from a 3-author team on public data.", "d": "Released embeddings support speaker identification in a pilot, with spoofing risk noted." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and Tier C release embedding models; Tier B tests them on Indonesian voices; Tier D uses them inside diarization and caller verification pilots.", "sources": [ "https://arxiv.org/abs/2005.07143", "https://arxiv.org/abs/2110.04410", "https://arxiv.org/abs/2303.00332" ], "unverified": [], "papers": [ { "title": "Generalized End-to-End Loss for Speaker Verification", "authors": "Li Wan et al.", "year": 2017, "venue": "ICASSP 2018 (per arXiv comment)", "url": "https://arxiv.org/abs/1710.10467", "role": "founding" }, { "title": "VoxCeleb: a large-scale speaker identification dataset", "authors": "Arsha Nagrani et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1706.08612", "role": "founding" }, { "title": "CAM++: A Fast and Efficient Network for Speaker Verification Using Context-Aware Masking", "authors": "Hui Wang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2303.00332", "role": "state of the art" }, { "title": "ECAPA-TDNN: Emphasized Channel Attention, Propagation and Aggregation in TDNN Based Speaker Verification", "authors": "Brecht Desplanques et al.", "year": 2020, "venue": "INTERSPEECH 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2005.07143", "role": "state of the art" }, { "title": "In defence of metric learning for speaker recognition", "authors": "Joon Son Chung et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2003.11982", "role": "state of the art" } ], "approved": true }, { "id": "massively-multilingual-machine-translation", "name": "Massively multilingual machine translation", "definition": "Dedicated translation models that cover hundreds of languages, many of them low-resource, in one system.", "compute_anchor": "MADLAD-400: a 10.7B parameter translation model trained on 250 billion tokens; NLLB uses a sparsely gated mixture of experts.", "data_anchor": "MADLAD-400: 3T token audited monolingual corpus in 419 languages; NLLB: mined data for 200 languages, evaluated on over 40,000 directions of Flores-200.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 39 listed on NLLB; 11 on MADLAD-400.", "time_horizon": "Years to build; months to fine-tune.", "shelf_life_signals": [ "LLM translators overtaking dedicated models on low-resource pairs" ], "exemplars": [ "meta-nllb-200", "google-madlad-400", "university-of-helsinki-opus-mt" ], "observed_actors": [ "meta", "google", "university-of-helsinki", "ai4bharat" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Meta and Google trained at 200 to 450 languages with teams of 11 to 39.", "b": "Training a 10.7B model on 250 billion tokens is a stretch, though fine-tuning NLLB for Indonesian regional languages is not.", "c": "Small per-pair OPUS-MT models trained on open corpora fit a university unit.", "d": "Released NLLB and OPUS-MT English-Indonesian models can be served on premises for document translation pilots." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A releases many-language models; Tier C and Tier B build Indonesian regional corpora and fine-tune; Tier D serves them for public sector translation.", "sources": [ "https://arxiv.org/abs/2207.04672", "https://arxiv.org/abs/2309.04662" ], "unverified": [], "papers": [ { "title": "Google's Multilingual Neural Machine Translation System: Enabling Zero-Shot Translation", "authors": "Melvin Johnson et al.", "year": 2016, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1611.04558", "role": "founding" }, { "title": "Massively Multilingual Neural Machine Translation in the Wild: Findings and Challenges", "authors": "Naveen Arivazhagan et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1907.05019", "role": "founding" }, { "title": "SeamlessM4T: Massively Multilingual & Multimodal Machine Translation", "authors": "Seamless Communication et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2308.11596", "role": "state of the art" }, { "title": "MADLAD-400: A Multilingual And Document-Level Large Audited Dataset", "authors": "Sneha Kudugunta et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2309.04662", "role": "state of the art" }, { "title": "No Language Left Behind: Scaling Human-Centered Machine Translation", "authors": "NLLB Team et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2207.04672", "role": "state of the art" } ], "approved": true }, { "id": "llm-based-machine-translation", "name": "LLM-based machine translation", "definition": "Specialising language models of a few billion parameters for translation through continued pre-training, instruction tuning and reinforcement learning.", "compute_anchor": "Seed-X and Hunyuan-MT reach their results at 7B parameters; training compute is not in the abstracts.", "data_anchor": "Seed-X: monolingual and bilingual data across 28 languages; Hunyuan-MT: 33 languages, first in 30 of 31 WMT2025 pairs.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 26 authors on Seed-X; 7 on Hunyuan-MT; 13 on Tower.", "time_horizon": "Months.", "shelf_life_signals": [ "General LLMs matching specialised 7B translators at similar cost" ], "exemplars": [ "bytedance-seed-x-ppo-7b", "tencent-hunyuan-mt-7b", "unbabel-towerinstruct-7b-v0-2", "google-translategemma" ], "observed_actors": [ "bytedance", "tencent", "unbabel", "google" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "ByteDance's 7B Seed-X is comparable to Gemini 2.5 and GPT-4o, and Tencent's Hunyuan-MT ranked first in 30 of 31 WMT2025 pairs.", "b": "Continued pre-training and RL at 7B, as Seed-X and Tower did, is the scale an Indonesian lab can run.", "c": "A unit can fine-tune a 7B model but the full Seed-X recipe used a 26-author team.", "d": "Open 7B translators can be served on premises for translation pilots." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A releases translation LLMs; Tier B adapts them to Indonesian and regional languages; Tier C evaluates; Tier D deploys them on premises.", "sources": [ "https://arxiv.org/abs/2507.13618", "https://arxiv.org/abs/2509.05209", "https://arxiv.org/abs/2402.17733" ], "unverified": [], "papers": [ { "title": "Is ChatGPT A Good Translator? Yes With GPT-4 As The Engine", "authors": "Wenxiang Jiao et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2301.08745", "role": "founding" }, { "title": "Tower: An Open Multilingual Large Language Model for Translation-Related Tasks", "authors": "Duarte M. Alves et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.17733", "role": "state of the art" }, { "title": "X-ALMA: Plug & Play Modules and Adaptive Rejection for Quality Translation at Scale", "authors": "Haoran Xu et al.", "year": 2024, "venue": "ICLR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2410.03115", "role": "state of the art" }, { "title": "Multilingual Machine Translation with Large Language Models: Empirical Results and Analysis", "authors": "Wenhao Zhu et al.", "year": 2023, "venue": "NAACL 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2304.04675", "role": "state of the art" }, { "title": "A Paradigm Shift in Machine Translation: Boosting Translation Performance of Large Language Models", "authors": "Haoran Xu et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2309.11674", "role": "state of the art" } ], "approved": true }, { "id": "speech-to-speech-translation", "name": "Speech-to-speech translation", "definition": "Translating spoken language directly into speech or text in another language, in one model rather than a cascade.", "compute_anchor": "Not in the sources read: the SeamlessM4T abstract gives no parameter count or compute.", "data_anchor": "SeamlessM4T: 1 million hours of open speech audio for w2v-BERT 2.0 pre-training; up to 100 languages.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 68 listed on SeamlessM4T.", "time_horizon": "Months to a year.", "shelf_life_signals": [ "Real-time voice models translating speech with natural latency" ], "exemplars": [ "meta-seamless-m4t", "openai-gpt-realtime-translate" ], "observed_actors": [ "meta", "openai", "nvidia" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "not feasible", "tier_d": "feasible", "tier_rationale": { "a": "Meta trained SeamlessM4T with 1 million hours of audio and 68 contributors.", "b": "A national lab could fine-tune SeamlessM4T but not repeat its 1 million hour pre-training.", "c": "A unit cannot match a 68-person effort on 1 million hours, though it can evaluate the released model.", "d": "Released and API speech translation models can run in interpreting pilots." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A releases speech translation models; Tier B and Tier C evaluate and tune them for Indonesian; Tier D pilots live interpreting.", "sources": [ "https://arxiv.org/abs/2308.11596" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Direct speech-to-speech translation with a sequence-to-sequence model", "authors": "Ye Jia et al.", "year": 2019, "venue": "Interspeech 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1904.06037", "role": "founding" }, { "title": "SeamlessM4T: Massively Multilingual & Multimodal Machine Translation", "authors": "Seamless Communication et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2308.11596", "role": "state of the art" }, { "title": "Seamless: Multilingual Expressive and Streaming Speech Translation", "authors": "Seamless Communication et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.05187", "role": "state of the art" }, { "title": "Direct speech-to-speech translation with discrete units", "authors": "Ann Lee et al.", "year": 2021, "venue": "ACL 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2107.05604", "role": "state of the art" }, { "title": "Recent Advances in Direct Speech-to-text Translation", "authors": "Chen Xu et al.", "year": 2023, "venue": "IJCAI2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2306.11646", "role": "survey" } ], "approved": true }, { "id": "multilingual-text-encoders", "name": "Multilingual and regional text encoders", "definition": "Text encoders pre-trained on many languages, or on one region's languages, for cross-lingual classification, tagging and retrieval.", "compute_anchor": "Not in the sources read: XLM-R, mmBERT and MuRIL abstracts give no training compute.", "data_anchor": "XLM-R: over two terabytes of filtered CommonCrawl in 100 languages; mmBERT: 3T tokens in over 1,800 languages.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 10 authors on XLM-R; 6 on mmBERT; 14 on MuRIL.", "time_horizon": "Months.", "shelf_life_signals": [ "Multilingual embedding models covering Indonesian regional languages well" ], "exemplars": [ "meta-xlm-roberta", "jhu-clsp-mmbert", "google-muril", "flax-community-indonesian-roberta-base" ], "observed_actors": [ "meta", "johns-hopkins-university", "google", "inria", "flax-community" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Meta trained XLM-R on two terabytes of text in 100 languages.", "b": "MuRIL shows a regional encoder for 17 Indian languages, a pattern an Indonesian lab can follow.", "c": "mmBERT came from a 6-author academic team.", "d": "Released multilingual encoders fine-tune for Indonesian classification in proofs of concept." }, "tier_confidence": { "a": "high", "b": "high", "c": "medium", "d": "high" }, "pipeline_role": "Tier A and Tier C release multilingual encoders; Tier B builds Indonesian and regional ones; Tier D fine-tunes them for local text tasks.", "sources": [ "https://arxiv.org/abs/1911.02116", "https://arxiv.org/abs/2509.06888", "https://arxiv.org/abs/2103.10730" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding", "authors": "Jacob Devlin et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1810.04805", "role": "founding" }, { "title": "Cross-lingual Language Model Pretraining", "authors": "Guillaume Lample et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1901.07291", "role": "founding" }, { "title": "Glot500: Scaling Multilingual Corpora and Language Models to 500 Languages", "authors": "Ayyoob Imani et al.", "year": 2023, "venue": "ACL 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2305.12182", "role": "state of the art" }, { "title": "XLM-V: Overcoming the Vocabulary Bottleneck in Multilingual Masked Language Models", "authors": "Davis Liang et al.", "year": 2023, "venue": "EMNLP 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2301.10472", "role": "state of the art" }, { "title": "mT5: A massively multilingual pre-trained text-to-text transformer", "authors": "Linting Xue et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2010.11934", "role": "state of the art" } ], "approved": true }, { "id": "compact-encoders-distillation", "name": "Compact text encoders through distillation", "definition": "Shrinking pre-trained text encoders through knowledge distillation and bottleneck designs so they run fast on servers and phones.", "compute_anchor": "DistilBERT is 40% smaller and 60% faster while keeping 97% of BERT's language understanding; MobileBERT runs at 62 ms on a Pixel 4.", "data_anchor": "Not in the sources read: DistilBERT, MobileBERT and MiniLM abstracts do not state pre-training data size.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on DistilBERT; 6 on MiniLM.", "time_horizon": "Weeks to months.", "shelf_life_signals": [ "Small decoder models overtaking distilled encoders on device" ], "exemplars": [ "hugging-face-distilbert", "google-mobilebert", "microsoft-minilm" ], "observed_actors": [ "hugging-face", "google", "microsoft" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google and Microsoft built MobileBERT and MiniLM.", "b": "Distilling an encoder is cheaper than pre-training it, as DistilBERT shows.", "c": "DistilBERT came from a 4-author team.", "d": "A 62 ms on-phone encoder fits on-device classification pilots." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and Tier C publish distillation methods; Tier B distils Indonesian encoders; Tier D deploys them on devices and low-cost servers.", "sources": [ "https://arxiv.org/abs/1910.01108", "https://arxiv.org/abs/2004.02984", "https://arxiv.org/abs/2002.10957" ], "unverified": [ "data_anchor" ], "papers": [ { "title": "Distilling the Knowledge in a Neural Network", "authors": "Geoffrey Hinton et al.", "year": 2015, "venue": "NIPS 2014 (per arXiv comment)", "url": "https://arxiv.org/abs/1503.02531", "role": "founding" }, { "title": "MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers", "authors": "Wenhui Wang et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2002.10957", "role": "state of the art" }, { "title": "MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices", "authors": "Zhiqing Sun et al.", "year": 2020, "venue": "ACL 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2004.02984", "role": "state of the art" }, { "title": "DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter", "authors": "Victor Sanh et al.", "year": 2019, "venue": "NeurIPS 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1910.01108", "role": "state of the art" }, { "title": "TinyBERT: Distilling BERT for Natural Language Understanding", "authors": "Xiaoqi Jiao et al.", "year": 2019, "venue": "EMNLP 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/1909.10351", "role": "state of the art" } ], "approved": true }, { "id": "zero-shot-extraction-small-encoders", "name": "Zero-shot extraction and classification with small encoders", "definition": "Using compact encoders to extract entities or assign labels named at run time, as a cheaper alternative to large language models.", "compute_anchor": "LLMLingua-2 uses XLM-RoBERTa-large and mBERT and is 3x to 6x faster than earlier prompt compression; GLiNER gives no compute.", "data_anchor": "Yin et al. unify zero-shot text classification as textual entailment; LLMLingua-2 builds an extractive compression dataset distilled from an LLM.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on GLiNER; 3 on the zero-shot classification benchmark.", "time_horizon": "Weeks to months.", "shelf_life_signals": [ "Small LLMs becoming as cheap as encoders for extraction" ], "exemplars": [ "urchade-gliner", "meta-bart-large-mnli", "microsoft-llmlingua-2" ], "observed_actors": [ "urchade", "meta", "microsoft" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Microsoft distilled knowledge from an LLM into an XLM-RoBERTa-large encoder for LLMLingua-2.", "b": "GLiNER beats ChatGPT zero-shot on NER with a compact encoder, cheap for a national lab to adapt to Indonesian.", "c": "GLiNER came from a 4-author team.", "d": "Zero-shot extraction lets a proof of concept pull entities from client documents without labelled data." }, "tier_confidence": { "a": "medium", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier C and Tier A publish compact extractors; Tier B adapts them to Indonesian; Tier D uses them for document triage and extraction.", "sources": [ "https://arxiv.org/abs/2311.08526", "https://arxiv.org/abs/2403.12968", "https://arxiv.org/abs/1909.00161" ], "unverified": [], "papers": [ { "title": "Benchmarking Zero-shot Text Classification: Datasets, Evaluation and Entailment Approach", "authors": "Wenpeng Yin et al.", "year": 2019, "venue": "EMNLP2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1909.00161", "role": "founding" }, { "title": "GLiREL -- Generalist Model for Zero-Shot Relation Extraction", "authors": "Jack Boylan et al.", "year": 2025, "venue": "NAACL 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2501.03172", "role": "state of the art" }, { "title": "NuNER: Entity Recognition Encoder Pre-training via LLM-Annotated Data", "authors": "Sergei Bogdanov et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.15343", "role": "state of the art" }, { "title": "UniversalNER: Targeted Distillation from Large Language Models for Open Named Entity Recognition", "authors": "Wenxuan Zhou et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2308.03279", "role": "state of the art" }, { "title": "GLiNER: Generalist Model for Named Entity Recognition using Bidirectional Transformer", "authors": "Urchade Zaratiana et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2311.08526", "role": "state of the art" } ], "approved": true }, { "id": "vision-language-action-robot-policies", "name": "Vision-language-action robot policies", "definition": "Robot policies that start from a pretrained vision-language model and output motor actions from camera images and a language instruction.", "compute_anchor": "OpenVLA: 7B parameters built on Llama 2 with DINOv2 and SigLIP features; fine-tunable on consumer GPUs with low-rank adaptation; pretraining compute not stated in the abstract.", "data_anchor": "OpenVLA: 970k real-world robot demonstrations; pi0: a large mixture from single-arm, dual-arm and mobile manipulators; GR00T N1: real-robot trajectories, human videos and synthetic data.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 18 authors on the OpenVLA paper (arXiv 2406.09246); 24 on pi0; 43 on GR00T N1.", "time_horizon": "Multi-year.", "shelf_life_signals": [ "OpenVLA (7B) beat the closed RT-2-X (55B) by 16.5 points across 29 tasks, so open models are overtaking closed ones within a year.", "SmolVLA reports performance comparable to VLAs 10x larger, so small open VLAs date quickly." ], "exemplars": [ "stanford-university-openvla", "physical-intelligence-pi0", "nvidia-gr00t-n1", "google-deepmind-gemini-robotics-1-5" ], "observed_actors": [ "stanford-university", "uc-berkeley", "physical-intelligence", "nvidia", "google-deepmind", "figure-ai", "agibot", "galbot", "unitree", "hugging-face", "baai", "allen-ai" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Physical Intelligence, NVIDIA and Google DeepMind train generalist VLAs on large multi-robot mixtures such as OpenVLA's 970k demonstrations.", "b": "Pretraining a VLA on 970k demonstrations is out of reach on the evidence, but open 7B weights can be fine-tuned with low-rank adaptation on consumer GPUs.", "c": "A lab with one robot arm can fine-tune open OpenVLA or pi0 weights on its own demonstrations, but collecting robot data needs hardware.", "d": "Fine-tuning an open VLA on consumer GPUs for one site task, such as retail picking, fits a proof of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A releases open VLA weights; Tier B and Tier C fine-tune them on local robots and tasks; Tier D carries a fine-tuned policy into a site proof of value.", "sources": [ "https://arxiv.org/abs/2406.09246", "https://arxiv.org/abs/2410.24164", "https://arxiv.org/abs/2503.14734", "https://arxiv.org/abs/2506.01844", "https://arxiv.org/abs/2507.02029" ], "unverified": [], "papers": [ { "title": "RT-1: Robotics Transformer for Real-World Control at Scale", "authors": "Anthony Brohan et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2212.06817", "role": "founding" }, { "title": "Gemini Robotics: Bringing AI into the Physical World", "authors": "Gemini Robotics Team et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2503.20020", "role": "state of the art" }, { "title": "Octo: An Open-Source Generalist Robot Policy", "authors": "Octo Model Team et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2405.12213", "role": "state of the art" }, { "title": "OpenVLA: An Open-Source Vision-Language-Action Model", "authors": "Moo Jin Kim et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.09246", "role": "state of the art" }, { "title": "A Survey on Vision-Language-Action Models for Embodied AI", "authors": "Yueen Ma et al.", "year": 2024, "venue": "IEEE Transactions on Neural Networks and Learning Systems (Early Access), 2026", "url": "https://arxiv.org/abs/2405.14093", "role": "survey" } ], "approved": true }, { "id": "imitation-learning-from-demonstrations", "name": "Imitation learning from demonstrations", "definition": "Training robot or game-playing policies directly from recorded human demonstrations, without a language model backbone.", "compute_anchor": "ACT runs on a low-cost bimanual hardware setup; Diffusion Policy and ACT training compute not stated in the abstracts; NitroGen scales behaviour cloning to 40,000 hours of video.", "data_anchor": "ACT: 6 real-world tasks learned at 80 to 90% success from about 10 minutes of demonstrations each; NitroGen: 40,000 hours of gameplay video across more than 1,000 games.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on the ACT paper (arXiv 2304.13705); 8 on Diffusion Policy.", "time_horizon": "Months.", "shelf_life_signals": [ "Diffusion Policy reports a 46.9% average improvement over prior methods across 12 tasks, then OpenVLA reports beating Diffusion Policy by 20.4% after fine-tuning." ], "exemplars": [ "act-action-chunking-transformer", "diffusion-policy" ], "observed_actors": [ "stanford-university", "columbia-university", "nyu", "nvidia", "tsinghua-university" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "NVIDIA scales behaviour cloning to 40,000 hours of gameplay video in NitroGen.", "b": "ACT learns real tasks from about 10 minutes of demonstrations on low-cost hardware, which fits a modest lab budget.", "c": "ACT learns real tasks from about 10 minutes of demonstrations on low-cost hardware, which fits a university robotics lab.", "d": "Ten minutes of demonstrations per task on low-cost hardware fits a site proof of concept for a repetitive manipulation task." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A scales demonstration data; Tier B and Tier C reproduce ACT and Diffusion Policy on local arms; Tier D uses them for single-task proofs of concept.", "sources": [ "https://arxiv.org/abs/2304.13705", "https://arxiv.org/abs/2303.04137", "https://arxiv.org/abs/2601.02427", "https://arxiv.org/abs/2406.09246" ], "unverified": [], "papers": [ { "title": "A Reduction of Imitation Learning and Structured Prediction to No-Regret Online Learning", "authors": "Stephane Ross et al.", "year": 2010, "venue": "AISTATS 2011 (per arXiv comment)", "url": "https://arxiv.org/abs/1011.0686", "role": "founding" }, { "title": "Behavioral Cloning from Observation", "authors": "Faraz Torabi et al.", "year": 2018, "venue": "IJCAI 2018 (per arXiv comment)", "url": "https://arxiv.org/abs/1805.01954", "role": "founding" }, { "title": "Mobile ALOHA: Learning Bimanual Mobile Manipulation with Low-Cost Whole-Body Teleoperation", "authors": "Zipeng Fu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2401.02117", "role": "state of the art" }, { "title": "Diffusion Policy: Visuomotor Policy Learning via Action Diffusion", "authors": "Cheng Chi et al.", "year": 2023, "venue": "RSS2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2303.04137", "role": "state of the art" }, { "title": "An Algorithmic Perspective on Imitation Learning", "authors": "Takayuki Osa et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1811.06711", "role": "survey" } ], "approved": true }, { "id": "humanoid-whole-body-control", "name": "Humanoid whole-body control", "definition": "Learned controllers that move a whole humanoid body, balancing, walking and manipulating at once.", "compute_anchor": "SONIC: 1.2M to 42M parameters trained with 21k GPU hours.", "data_anchor": "SONIC: over 100M frames from 700 hours of motion capture.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 28 authors on the SONIC paper (arXiv 2511.07820).", "time_horizon": "Multi-year.", "shelf_life_signals": [ "SONIC notes that earlier humanoid controllers were trained on a handful of GPUs; scaling to 21k GPU hours changed the state of the art." ], "exemplars": [ "nvidia-gear-sonic", "figure-ai-helix-02" ], "observed_actors": [ "nvidia", "figure-ai", "galbot", "tesla", "google-deepmind" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "NVIDIA trained SONIC with 21k GPU hours on 700 hours of motion capture.", "b": "21k GPU hours and a humanoid robot are beyond the evidence for Indonesian labs, though the model itself is only up to 42M parameters.", "c": "A 42M-parameter controller is small, but 21k GPU hours and humanoid hardware are not shown for institution units.", "d": "Building a whole-body controller needs 21k GPU hours and humanoid hardware; delivery work would only use vendor robots." }, "tier_confidence": { "a": "high", "b": "low", "c": "low", "d": "medium" }, "pipeline_role": "Tier A trains whole-body controllers on its humanoids; Tier B and Tier C could study released controllers in simulation; Tier D would deploy vendor humanoids rather than build controllers.", "sources": [ "https://arxiv.org/abs/2511.07820", "https://arxiv.org/abs/2503.14734" ], "unverified": [], "papers": [ { "title": "DeepMimic: Example-Guided Deep Reinforcement Learning of Physics-Based Character Skills", "authors": "Xue Bin Peng et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1804.02717", "role": "founding" }, { "title": "GR00T N1: An Open Foundation Model for Generalist Humanoid Robots", "authors": "NVIDIA et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2503.14734", "role": "state of the art" }, { "title": "Expressive Whole-Body Control for Humanoid Robots", "authors": "Xuxin Cheng et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.16796", "role": "state of the art" }, { "title": "Learning Human-to-Humanoid Real-Time Whole-Body Teleoperation", "authors": "Tairan He et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.04436", "role": "state of the art" }, { "title": "OmniH2O: Universal and Dexterous Human-to-Humanoid Whole-Body Teleoperation and Learning", "authors": "Tairan He et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.08858", "role": "state of the art" } ], "approved": true }, { "id": "robot-perception-for-manipulation-and-navigation", "name": "Robot perception for manipulation and navigation", "definition": "Models that give a robot the 3D facts it needs to act: object pose, depth, touch and its own location.", "compute_anchor": "FoundationPose: training compute not stated in the abstract; trained at large scale on synthetic data.", "data_anchor": "FoundationPose: large-scale synthetic training aided by a large language model; works on novel objects from a CAD model or a few reference images.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on the FoundationPose paper (arXiv 2312.08344).", "time_horizon": "Months.", "shelf_life_signals": [ "FoundationPose reports beating methods specialised for each setup, which retires per-object pose models." ], "exemplars": [ "nvidia-foundationpose" ], "observed_actors": [ "nvidia", "meta", "eth-zurich" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "NVIDIA trains FoundationPose on large-scale synthetic data.", "b": "FoundationPose runs on a novel object from its CAD model without fine-tuning, so use needs little compute.", "c": "Applying FoundationPose to new objects needs only a CAD model or a few reference images.", "d": "Pose estimation from a CAD model without fine-tuning fits a proof of concept for robotic picking or inspection." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A releases perception models; Tier B and Tier C test them on local objects; Tier D builds picking or inspection proofs of concept on them.", "sources": [ "https://arxiv.org/abs/2312.08344" ], "unverified": [], "papers": [ { "title": "Dex-Net 2.0: Deep Learning to Plan Robust Grasps with Synthetic Point Clouds and Analytic Grasp Metrics", "authors": "Jeffrey Mahler et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1703.09312", "role": "founding" }, { "title": "ViNT: A Foundation Model for Visual Navigation", "authors": "Dhruv Shah et al.", "year": 2023, "venue": "CoRL 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2306.14846", "role": "state of the art" }, { "title": "AnyGrasp: Robust and Efficient Grasp Perception in Spatial and Temporal Domains", "authors": "Hao-Shu Fang et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2212.08333", "role": "state of the art" }, { "title": "ORB-SLAM3: An Accurate Open-Source Library for Visual, Visual-Inertial and Multi-Map SLAM", "authors": "Carlos Campos et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2007.11898", "role": "state of the art" }, { "title": "Habitat: A Platform for Embodied AI Research", "authors": "Manolis Savva et al.", "year": 2019, "venue": "ICCV 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1904.01201", "role": "state of the art" } ], "approved": true }, { "id": "model-based-rl-with-learned-world-models", "name": "Model-based reinforcement learning with learned world models", "definition": "Agents that learn a model of their environment and improve by planning or imagining inside it, rather than generating worlds for people.", "compute_anchor": "DreamerV3: training compute not stated in the abstract; one configuration across over 150 tasks.", "data_anchor": "DreamerV3: collects Minecraft diamonds from scratch without human data or curricula.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on the DreamerV3 paper (arXiv 2301.04104); 12 on MuZero (arXiv 1911.08265).", "time_horizon": "Multi-year.", "shelf_life_signals": [ "MuZero plans with a learned model without known rules, and DreamerV3 uses one configuration across over 150 tasks." ], "exemplars": [ "google-deepmind-dreamerv3", "google-deepmind-muzero" ], "observed_actors": [ "google-deepmind" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Google DeepMind develops both Dreamer and MuZero.", "b": "DreamerV3 runs with one configuration across over 150 tasks, but compute is not stated in the abstract.", "c": "A fixed configuration across over 150 tasks makes DreamerV3 a usable research baseline for university labs.", "d": "No industrial use is shown in the sources; use in a proof of concept would be exploratory." }, "tier_confidence": { "a": "high", "b": "low", "c": "medium", "d": "low" }, "pipeline_role": "Tier A publishes algorithms; Tier C reproduces them on simulated tasks; Tier B and Tier D have no observed role yet.", "sources": [ "https://arxiv.org/abs/2301.04104", "https://arxiv.org/abs/1911.08265" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "World Models", "authors": "David Ha et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1803.10122", "role": "founding" }, { "title": "Mastering Diverse Domains through World Models", "authors": "Danijar Hafner et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2301.04104", "role": "state of the art" }, { "title": "TD-MPC2: Scalable, Robust World Models for Continuous Control", "authors": "Nicklas Hansen et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2310.16828", "role": "state of the art" }, { "title": "Dream to Control: Learning Behaviors by Latent Imagination", "authors": "Danijar Hafner et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1912.01603", "role": "state of the art" }, { "title": "Model-based Reinforcement Learning: A Survey", "authors": "Thomas M. Moerland et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2006.16712", "role": "survey" } ], "approved": true }, { "id": "driving-world-models-and-simulation", "name": "Driving world models and closed-loop simulation", "definition": "Generative models of road scenes that predict how traffic responds to the vehicle's actions, used to train and test driving systems.", "compute_anchor": "GAIA-1: compute and parameter count not stated in the abstract.", "data_anchor": "GAIA-1: video, text and action inputs mapped to discrete tokens and modelled as next-token prediction.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 8 authors on the GAIA-1 paper (arXiv 2309.17080).", "time_horizon": "Multi-year.", "shelf_life_signals": [ "Waymo built the Waymo World Model on Genie 3, a general world model, rather than on a driving-only model." ], "exemplars": [ "wayve-gaia-1", "wayve-gaia-3", "waymo-world-model" ], "observed_actors": [ "wayve", "waymo", "tesla", "nvidia", "huawei" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "stretch", "tier_rationale": { "a": "Wayve built GAIA-1 and Waymo built its world model on Genie 3.", "b": "The GAIA-1 abstract gives no compute or data size and the sources show only fleet operators as builders.", "c": "Institution units could evaluate open NVIDIA Cosmos world models for driving, but no compute anchor is sourced.", "d": "Delivery work could use a vendor driving world model to test scenarios such as mine haulage, but none is shown open for that use." }, "tier_confidence": { "a": "high", "b": "low", "c": "low", "d": "low" }, "pipeline_role": "Tier A fleet operators build driving world models; Tier C could evaluate open ones; Tier D could use vendor simulators for scenario testing in mining or logistics.", "sources": [ "https://arxiv.org/abs/2309.17080", "https://en.wikipedia.org/wiki/Genie_(AI_model)" ], "unverified": [ "compute_anchor", "data_anchor" ], "papers": [ { "title": "GAIA-1: A Generative World Model for Autonomous Driving", "authors": "Anthony Hu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2309.17080", "role": "founding" }, { "title": "GenAD: Generalized Predictive Model for Autonomous Driving", "authors": "Jiazhi Yang et al.", "year": 2024, "venue": "CVPR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2403.09630", "role": "state of the art" }, { "title": "Vista: A Generalizable Driving World Model with High Fidelity and Versatile Controllability", "authors": "Shenyuan Gao et al.", "year": 2024, "venue": "NeurIPS 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2405.17398", "role": "state of the art" }, { "title": "DriveDreamer: Towards Real-world-driven World Models for Autonomous Driving", "authors": "Xiaofeng Wang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2309.09777", "role": "state of the art" }, { "title": "World Models for Autonomous Driving: An Initial Survey", "authors": "Yanchen Guan et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.02622", "role": "survey" } ], "approved": true }, { "id": "end-to-end-driving-policies-with-reasoning", "name": "End-to-end driving policies with reasoning", "definition": "Driving models that map sensor input straight to a planned trajectory, increasingly with a language model that reasons about the scene first.", "compute_anchor": "Alpamayo-R1: scaled from 0.5B to 7B parameters on the Cosmos-Reason backbone; 99 ms on-vehicle latency.", "data_anchor": "Alpamayo-R1: Chain of Causation dataset built by auto-labelling plus human review; RL post-training.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 44 authors on the Alpamayo-R1 paper (arXiv 2511.00088).", "time_horizon": "Multi-year.", "shelf_life_signals": [ "Alpamayo-R1 reports a 35% lower close-encounter rate in closed-loop simulation than a trajectory-only baseline." ], "exemplars": [ "nvidia-alpamayo-r1", "tesla-fsd-v14" ], "observed_actors": [ "nvidia", "tesla" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "NVIDIA trains Alpamayo-R1 at 0.5B to 7B parameters and Tesla trains FSD on its fleet.", "b": "Building a driving VLA needs a curated reasoning dataset and on-vehicle testing that the sources show only for NVIDIA and Tesla.", "c": "Open Alpamayo-R1 weights allow evaluation in simulation, but on-vehicle tests are not shown for institution units.", "d": "Road-vehicle autonomy is not a proof-of-concept scope; delivery work would stop at evaluation." }, "tier_confidence": { "a": "high", "b": "medium", "c": "low", "d": "medium" }, "pipeline_role": "Tier A trains and releases driving VLAs; Tier C evaluates open weights in simulation; Tier B and Tier D have no observed role.", "sources": [ "https://arxiv.org/abs/2511.00088" ], "unverified": [], "papers": [ { "title": "ChauffeurNet: Learning to Drive by Imitating the Best and Synthesizing the Worst", "authors": "Mayank Bansal et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1812.03079", "role": "founding" }, { "title": "Alpamayo-R1: Bridging Reasoning and Action Prediction for Generalizable Autonomous Driving in the Long Tail", "authors": "NVIDIA et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2511.00088", "role": "state of the art" }, { "title": "EMMA: End-to-End Multimodal Model for Autonomous Driving", "authors": "Jyh-Jing Hwang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2410.23262", "role": "state of the art" }, { "title": "VAD: Vectorized Scene Representation for Efficient Autonomous Driving", "authors": "Bo Jiang et al.", "year": 2023, "venue": "ICCV 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2303.12077", "role": "state of the art" }, { "title": "End-to-end Autonomous Driving: Challenges and Frontiers", "authors": "Li Chen et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.16927", "role": "survey" } ], "approved": true }, { "id": "multi-camera-bev-3d-perception", "name": "Multi-camera bird's-eye-view 3D perception", "definition": "Turning several camera views around a vehicle into one top-down map for 3D object detection and map segmentation.", "compute_anchor": "BEVFormer: training compute not stated in the abstract.", "data_anchor": "BEVFormer: evaluated on the nuScenes test set at 56.9% NDS.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 8 authors on the BEVFormer paper (arXiv 2203.17270).", "time_horizon": "Years.", "shelf_life_signals": [ "End-to-end driving models such as Alpamayo-R1 now plan from cameras directly, which reduces the role of a separate BEV stage." ], "exemplars": [ "bevformer", "bevdet" ], "observed_actors": [ "nanjing-university", "shanghai-ai-lab", "phigent-robotics", "ut-austin", "stanford-university", "carnegie-mellon-university", "hust" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Shanghai AI Lab and university partners publish BEVFormer at 56.9% NDS on nuScenes.", "b": "Open code and the public nuScenes benchmark make reproduction possible, though compute is not stated in the abstract.", "c": "University labs can train on nuScenes with open code; compute is not stated in the abstract.", "d": "Multi-camera BEV perception needs a calibrated camera rig, which narrows proofs of concept to site vehicles." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "low" }, "pipeline_role": "Tier A and university labs publish BEV methods; Tier B and Tier C reproduce them on local data; Tier D could use them for site vehicles such as mine trucks.", "sources": [ "https://arxiv.org/abs/2203.17270" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Lift, Splat, Shoot: Encoding Images From Arbitrary Camera Rigs by Implicitly Unprojecting to 3D", "authors": "Jonah Philion et al.", "year": 2020, "venue": "ECCV 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2008.05711", "role": "founding" }, { "title": "BEVFormer: Learning Bird's-Eye-View Representation from Multi-Camera Images via Spatiotemporal Transformers", "authors": "Zhiqi Li et al.", "year": 2022, "venue": "ECCV 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2203.17270", "role": "state of the art" }, { "title": "BEVFusion: Multi-Task Multi-Sensor Fusion with Unified Bird's-Eye View Representation", "authors": "Zhijian Liu et al.", "year": 2022, "venue": "ICRA 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2205.13542", "role": "state of the art" }, { "title": "PETR: Position Embedding Transformation for Multi-View 3D Object Detection", "authors": "Yingfei Liu et al.", "year": 2022, "venue": "ECCV 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2203.05625", "role": "state of the art" }, { "title": "Delving into the Devils of Bird's-eye-view Perception: A Review, Evaluation and Recipe", "authors": "Hongyang Li et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2209.05324", "role": "survey" } ], "approved": true }, { "id": "real-time-road-and-lidar-segmentation", "name": "Real-time road, lane and LiDAR segmentation on embedded hardware", "definition": "Light perception networks that find vehicles, drivable area, lanes and LiDAR classes fast enough for embedded boards in vehicles.", "compute_anchor": "YOLOP: 23 FPS on an embedded Jetson TX2.", "data_anchor": "YOLOP: BDD100K driving dataset; SalsaNext: LiDAR point clouds.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 7 authors on the YOLOP paper (arXiv 2108.11250).", "time_horizon": "Months.", "shelf_life_signals": [ "HybridNets followed YOLOP within a year on the same three tasks." ], "exemplars": [ "hust-yolop", "hybridnets", "salsanext" ], "observed_actors": [ "hust", "ku-leuven", "halmstad-university", "volvo", "intel" ], "tier_a": "stretch", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs are not observed here; the actors are universities and chip vendors.", "b": "YOLOP runs at 23 FPS on a Jetson TX2 and trains on the public BDD100K dataset.", "c": "YOLOP runs at 23 FPS on a Jetson TX2 and trains on the public BDD100K dataset.", "d": "A 23 FPS model on an embedded board fits a fleet safety or site vehicle proof of concept." }, "tier_confidence": { "a": "medium", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Universities publish light networks; Tier B and Tier C retrain them on local roads; Tier D fits them to fleet safety and site vehicles.", "sources": [ "https://arxiv.org/abs/2108.11250", "https://arxiv.org/abs/2203.09035" ], "unverified": [], "papers": [ { "title": "SqueezeSeg: Convolutional Neural Nets with Recurrent CRF for Real-Time Road-Object Segmentation from 3D LiDAR Point Cloud", "authors": "Bichen Wu et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1710.07368", "role": "founding" }, { "title": "Point Transformer V3: Simpler, Faster, Stronger", "authors": "Xiaoyang Wu et al.", "year": 2023, "venue": "CVPR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2312.10035", "role": "state of the art" }, { "title": "2DPASS: 2D Priors Assisted Semantic Segmentation on LiDAR Point Clouds", "authors": "Xu Yan et al.", "year": 2022, "venue": "ECCV2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2207.04397", "role": "state of the art" }, { "title": "PolarNet: An Improved Grid Representation for Online LiDAR Point Clouds Semantic Segmentation", "authors": "Yang Zhang et al.", "year": 2020, "venue": "CVPR 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2003.14032", "role": "state of the art" }, { "title": "Cylindrical and Asymmetrical 3D Convolution Networks for LiDAR Segmentation", "authors": "Xinge Zhu et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2011.10033", "role": "state of the art" } ], "approved": true }, { "id": "earth-observation-foundation-models", "name": "Earth observation foundation models", "definition": "Models pretrained on large satellite and aerial archives that are fine-tuned for land cover, crops, floods and other mapping tasks.", "compute_anchor": "Prithvi-EO-2.0: up to 600M parameters; training compute not stated in the abstract.", "data_anchor": "Prithvi-EO-2.0: 4.2 million global time-series samples from NASA Harmonized Landsat and Sentinel-2 at 30 m; AlphaEarth Foundations releases annual embedding layers for 2017 to 2024.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 36 authors on the Prithvi-EO-2.0 paper (arXiv 2412.02732); 19 on AlphaEarth Foundations.", "time_horizon": "Years.", "shelf_life_signals": [ "Prithvi-EO-2.0 beats Prithvi-EO-1.0 by 8% on GEO-Bench and six other geospatial foundation models." ], "exemplars": [ "ibm-prithvi-eo-2-0", "google-deepmind-alphaearth-foundations", "ant-group-skysense" ], "observed_actors": [ "ibm", "nasa", "google-deepmind", "ant-group", "aircas", "clay-foundation", "esa", "meta", "stanford-university" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "IBM with NASA, Google DeepMind and Ant Group pretrain EO models on archives such as 4.2 million HLS samples.", "b": "Open Prithvi-EO-2.0 weights and the open HLS archive allow fine-tuning on Indonesian land cover.", "c": "Fine-tuning a model of up to 600M parameters on labelled local tiles fits a university remote sensing lab.", "d": "AlphaEarth annual embeddings and open Prithvi weights fit plantation and land monitoring proofs of value with few labels." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A pretrains and releases EO models and embeddings; Tier B and Tier C fine-tune them for Indonesian land; Tier D builds monitoring proofs of value on them.", "sources": [ "https://arxiv.org/abs/2412.02732", "https://arxiv.org/abs/2507.22291" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "SatMAE: Pre-training Transformers for Temporal and Multi-Spectral Satellite Imagery", "authors": "Yezhen Cong et al.", "year": 2022, "venue": "NeurIPS 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2207.08051", "role": "founding" }, { "title": "AlphaEarth Foundations: An embedding field model for accurate and efficient global mapping from sparse label data", "authors": "Christopher F. Brown et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2507.22291", "role": "state of the art" }, { "title": "Neural Plasticity-Inspired Multimodal Foundation Model for Earth Observation", "authors": "Zhitong Xiong et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.15356", "role": "state of the art" }, { "title": "Lightweight, Pre-trained Transformers for Remote Sensing Timeseries", "authors": "Gabriel Tseng et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2304.14065", "role": "state of the art" }, { "title": "On the Foundations of Earth and Climate Foundation Models", "authors": "Xiao Xiang Zhu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2405.04285", "role": "survey" } ], "approved": true }, { "id": "plantation-and-forest-mapping", "name": "Plantation and natural forest mapping from satellite and drone imagery", "definition": "Mapping tree crops such as oil palm and separating natural forest from plantations, for yield, deforestation baselines and supply-chain rules such as the EUDR.", "compute_anchor": "Natural Forests of the World 2020 (MTSViT): compute not stated in the source.", "data_anchor": "MTSViT: seasonal Sentinel-2 imagery plus elevation and slope over 1280 m patches; 10 m map at 92.2% accuracy against an independent validation set. PySawit: vendor reports accuracy raised from 78% to up to 90% using 90 m weather data.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): Not in the sources read (blog posts give no author list).", "time_horizon": "Months.", "shelf_life_signals": [ "The EUDR 2020 cut-off makes a 2020 baseline map useful for years, but newer forest classes are planned by Google." ], "exemplars": [ "google-deepmind-natural-forests-mtsvit", "eosda-pysawit" ], "observed_actors": [ "google-deepmind", "google", "wri", "eosda" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google DeepMind with WRI built the 10 m natural forest map at 92.2% accuracy.", "b": "Sentinel-2 imagery is open and the 2020 natural forest map is public, so Indonesian labs can build on both.", "c": "Mapping with open Sentinel-2 data and public baselines fits a university lab.", "d": "EOSDA reports raising its PySawit oil palm model from 78% to up to 90% accuracy, and a public 10 m baseline supports EUDR proofs of value." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A publishes global baselines; Tier B and Tier C adapt them to Indonesian plantations; Tier D builds EUDR and yield proofs of value.", "sources": [ "https://research.google/blog/separating-natural-forests-from-other-tree-cover-with-ai-for-deforestation-free-supply-chains/", "https://eos.com/blog/eosda-enhanced-pysawit-model-to-improve-its-accuracy/" ], "unverified": [ "compute_anchor", "team_size_proxy" ], "papers": [ { "title": "A high-resolution canopy height model of the Earth", "authors": "Nico Lang et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2204.08322", "role": "founding" }, { "title": "Very high resolution canopy height maps from RGB imagery using self-supervised vision transformer and convolutional decoder trained on Aerial Lidar", "authors": "Jamie Tolan et al.", "year": 2023, "venue": "Remote Sensing of Environment 300, 113888, 2024", "url": "https://arxiv.org/abs/2304.07213", "role": "state of the art" }, { "title": "Mapping oil palm density at country scale: An active learning approach", "authors": "Andrés C. Rodríguez et al.", "year": 2021, "venue": "Remote Sensing of Environment Volume 261, August 2021, 112479", "url": "https://arxiv.org/abs/2105.11207", "role": "state of the art" }, { "title": "Satellite reveals age and extent of oil palm plantations in Southeast Asia", "authors": "Olha Danylo et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2002.07163", "role": "state of the art" }, { "title": "ForestNet: Classifying Drivers of Deforestation in Indonesia using Deep Learning on Satellite Imagery", "authors": "Jeremy Irvin et al.", "year": 2020, "venue": "NeurIPS 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2011.05479", "role": "state of the art" } ], "approved": true }, { "id": "oriented-object-detection-in-aerial-imagery", "name": "Oriented object detection in aerial imagery", "definition": "Detecting ships, planes, vehicles and tanks in overhead images with rotated boxes that follow each object's heading.", "compute_anchor": "Oriented R-CNN: ResNet50 backbone, 15.1 FPS at 1024x1024 on a single RTX 2080Ti; training compute not stated in the abstract.", "data_anchor": "Oriented R-CNN: DOTA (75.87% mAP) and HRSC2016 (96.50% mAP) benchmarks.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on the Oriented R-CNN paper (arXiv 2108.05699).", "time_horizon": "Months.", "shelf_life_signals": [ "Open toolkits such as MMRotate collect many detectors, so single methods are overtaken quickly." ], "exemplars": [ "oriented-r-cnn", "roi-transformer" ], "observed_actors": [ "nwpu", "wuhan-university", "shanghai-jiao-tong-university", "openmmlab" ], "tier_a": "stretch", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Frontier labs are not observed; the actors are universities and OpenMMLab.", "b": "A ResNet50 detector trained on public DOTA data fits a modest budget.", "c": "A ResNet50 detector trained on public DOTA data fits a university lab.", "d": "Rotated-box detection of vessels or tanks fits a satellite monitoring proof of concept." }, "tier_confidence": { "a": "medium", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Universities publish detectors; Tier B and Tier C retrain them on Indonesian imagery; Tier D applies them in monitoring proofs of concept.", "sources": [ "https://arxiv.org/abs/2108.05699" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "DOTA: A Large-scale Dataset for Object Detection in Aerial Images", "authors": "Gui-Song Xia et al.", "year": 2017, "venue": "IEEE Conference on Computer Vision and Pattern Recognition, 2018", "url": "https://arxiv.org/abs/1711.10398", "role": "founding" }, { "title": "Large Selective Kernel Network for Remote Sensing Object Detection", "authors": "Yuxuan Li et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2303.09030", "role": "state of the art" }, { "title": "Oriented R-CNN for Object Detection", "authors": "Xingxing Xie et al.", "year": 2021, "venue": "ICCV 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2108.05699", "role": "state of the art" }, { "title": "ReDet: A Rotation-equivariant Detector for Aerial Object Detection", "authors": "Jiaming Han et al.", "year": 2021, "venue": "CVPR2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2103.07733", "role": "state of the art" }, { "title": "Align Deep Features for Oriented Object Detection", "authors": "Jiaming Han et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2008.09397", "role": "state of the art" } ], "approved": true }, { "id": "image-geolocalisation-and-visual-place-recognition", "name": "Image geolocalisation and visual place recognition", "definition": "Finding where a photo was taken, either on the globe or against a map of known places.", "compute_anchor": "StreetCLIP and PlaNet: training compute not stated in the abstracts.", "data_anchor": "PlaNet: millions of geotagged images over thousands of geographic cells; StreetCLIP: zero-shot, beating supervised models trained on more than 4 million images.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on the PlaNet paper (arXiv 1602.05314); 3 on StreetCLIP.", "time_horizon": "Years.", "shelf_life_signals": [ "StreetCLIP reaches state of the art zero-shot by grounding CLIP, overtaking supervised classifiers such as PlaNet." ], "exemplars": [ "streetclip", "google-planet-v2" ], "observed_actors": [ "google", "stanford-university", "polytechnic-of-turin", "eth-zurich" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Google trained PlaNet on millions of geotagged images.", "b": "StreetCLIP is public and works zero-shot, so use needs little compute.", "c": "StreetCLIP is public and works zero-shot, so a university lab can evaluate it on local photos.", "d": "Field photo verification is a possible use, but no industrial deployment is shown in the sources." }, "tier_confidence": { "a": "medium", "b": "medium", "c": "high", "d": "low" }, "pipeline_role": "Tier A trains global models; Tier B and Tier C evaluate them on Indonesian imagery; Tier D could use them to check field photo locations.", "sources": [ "https://arxiv.org/abs/1602.05314", "https://arxiv.org/abs/2302.00275" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "NetVLAD: CNN architecture for weakly supervised place recognition", "authors": "Relja Arandjelović et al.", "year": 2015, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1511.07247", "role": "founding" }, { "title": "MixVPR: Feature Mixing for Visual Place Recognition", "authors": "Amar Ali-bey et al.", "year": 2023, "venue": "WACV 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2303.02190", "role": "state of the art" }, { "title": "GeoCLIP: Clip-Inspired Alignment between Locations and Images for Effective Worldwide Geo-localization", "authors": "Vicente Vivanco Cepeda et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2309.16020", "role": "state of the art" }, { "title": "AnyLoc: Towards Universal Visual Place Recognition", "authors": "Nikhil Keetha et al.", "year": 2023, "venue": "ICRA 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2308.00688", "role": "state of the art" }, { "title": "General Place Recognition Survey: Towards the Real-world Autonomy Age", "authors": "Peng Yin et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2209.04497", "role": "survey" } ], "approved": true }, { "id": "machine-learning-medium-range-weather-forecasting", "name": "Machine-learning medium-range weather forecasting", "definition": "Global forecast models learned from reanalysis data that predict the next 10 to 15 days much faster than physics models.", "compute_anchor": "Pangu-Weather: about 256 million parameters in total; GraphCast: a 10-day 0.25 degree forecast in under one minute; training compute not stated in the abstracts.", "data_anchor": "Pangu-Weather: 43 years of hourly ERA5; FuXi: 39 years of ERA5; Aurora: over a million hours of geophysical data.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 18 authors on the GraphCast paper (arXiv 2212.12794); 6 on Pangu-Weather.", "time_horizon": "Years.", "shelf_life_signals": [ "GraphCast beat the most accurate operational deterministic systems on 90% of 1380 targets; FuXi then matched the ECMWF ensemble mean at 15 days." ], "exemplars": [ "google-deepmind-graphcast", "huawei-pangu-weather", "fudan-university-fuxi", "microsoft-aurora", "ecmwf-aifs" ], "observed_actors": [ "google-deepmind", "huawei", "fudan-university", "shanghai-ai-lab", "microsoft", "nvidia", "ecmwf", "ibm", "nasa" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Google DeepMind and Huawei train on decades of ERA5 and beat operational physics models; Microsoft trains Aurora on over a million hours of data.", "b": "A 256M-parameter model on public ERA5 is within reach, though training compute is not stated in the abstracts.", "c": "Public ERA5 and open GraphCast code allow fine-tuning studies, though training compute is not stated.", "d": "Running a released model in under one minute per forecast fits plantation and energy forecasting proofs of value." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A trains global models; Tier B and Tier C fine-tune them for the Maritime Continent; Tier D runs released models for sector forecasts.", "sources": [ "https://arxiv.org/abs/2212.12794", "https://arxiv.org/abs/2211.02556", "https://arxiv.org/abs/2306.12873", "https://arxiv.org/abs/2405.13063" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "FourCastNet: A Global Data-driven High-resolution Weather Model using Adaptive Fourier Neural Operators", "authors": "Jaideep Pathak et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2202.11214", "role": "founding" }, { "title": "A Foundation Model for the Earth System", "authors": "Cristian Bodnar et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2405.13063", "role": "state of the art" }, { "title": "FuXi: A cascade machine learning forecasting system for 15-day global weather forecast", "authors": "Lei Chen et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.12873", "role": "state of the art" }, { "title": "WeatherBench 2: A benchmark for the next generation of data-driven global weather models", "authors": "Stephan Rasp et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2308.15560", "role": "state of the art" }, { "title": "Pangu-Weather: A 3D High-Resolution Model for Fast and Accurate Global Weather Forecast", "authors": "Kaifeng Bi et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2211.02556", "role": "state of the art" } ], "approved": true }, { "id": "probabilistic-ensemble-weather-forecasting", "name": "Probabilistic ensemble weather forecasting with machine learning", "definition": "Machine-learning models that produce many possible weather futures, giving forecast uncertainty instead of one answer.", "compute_anchor": "GenCast: a 15-day global ensemble at 0.25 degree for over 80 variables in 8 minutes; training compute not stated in the abstract.", "data_anchor": "GenCast: decades of reanalysis data; skill above ECMWF ENS on 97.4% of 1320 targets.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 12 authors on the GenCast paper (arXiv 2312.15796).", "time_horizon": "Years.", "shelf_life_signals": [ "GenCast already beats ECMWF ENS on 97.4% of 1320 targets, so the operational physics ensemble is no longer the bar." ], "exemplars": [ "google-deepmind-gencast", "google-deepmind-weathernext-2", "nvidia-fourcastnet-3" ], "observed_actors": [ "google-deepmind", "google", "ecmwf", "nvidia", "inria", "ucla" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Google DeepMind built GenCast, which beats ECMWF ENS on 97.4% of 1320 targets.", "b": "Ensembles multiply inference cost, though one 15-day ensemble takes 8 minutes; training compute is not stated.", "c": "Open GenCast code allows study, but training compute is not stated in the abstract.", "d": "Using released ensemble forecasts for risk ranges fits energy and agriculture proofs of value." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A trains ensembles; Tier B and Tier C verify them over Indonesia; Tier D turns forecast ranges into decision support.", "sources": [ "https://arxiv.org/abs/2312.15796" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "GenCast: Diffusion-based ensemble forecasting for medium-range weather", "authors": "Ilan Price et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.15796", "role": "founding" }, { "title": "FourCastNet 3: A geometric approach to probabilistic machine-learning weather forecasting at scale", "authors": "Boris Bonev et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2507.12144", "role": "state of the art" }, { "title": "FuXi-ENS: A machine learning model for medium-range ensemble weather forecasting", "authors": "Xiaohui Zhong et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2405.05925", "role": "state of the art" }, { "title": "ArchesWeather & ArchesWeatherGen: a deterministic and generative model for efficient ML weather forecasting", "authors": "Guillaume Couairon et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2412.12971", "role": "state of the art" }, { "title": "Neural General Circulation Models for Weather and Climate", "authors": "Dmitrii Kochkov et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2311.07222", "role": "state of the art" } ], "approved": true }, { "id": "km-scale-generative-downscaling", "name": "Kilometre-scale generative downscaling of weather", "definition": "Generative models that turn coarse global forecasts into kilometre-scale local fields for hazards such as heavy rain.", "compute_anchor": "CorrDiff: a UNet plus a corrector diffusion model; training compute not stated in the abstract.", "data_anchor": "CorrDiff: 2 km regional model data over Taiwan conditioned on 25 km global reanalysis.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 13 authors on the CorrDiff paper (arXiv 2309.15214).", "time_horizon": "Years.", "shelf_life_signals": [ "NVIDIA followed CorrDiff (arXiv 2309) with StormCast (arXiv 2408) and cBottle (arXiv 2505)." ], "exemplars": [ "nvidia-corrdiff", "nvidia-stormcast", "meteoswiss-varda-single-1-0" ], "observed_actors": [ "nvidia", "meteoswiss" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "stretch", "tier_d": "stretch", "tier_rationale": { "a": "NVIDIA built CorrDiff on 2 km Taiwan data.", "b": "CorrDiff needs paired km-scale regional model output, which a national weather agency holds for its own area.", "c": "Training needs km-scale regional data that institution units may not hold.", "d": "A downscaled rainfall field for one region fits a proof of technology, but needs regional training data." }, "tier_confidence": { "a": "high", "b": "medium", "c": "low", "d": "low" }, "pipeline_role": "Tier A publishes downscaling models; Tier B adapts them with national km-scale data; Tier D could use them for local hazard proofs.", "sources": [ "https://arxiv.org/abs/2309.15214" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "DeepSD: Generating High Resolution Climate Change Projections through Single Image Super-Resolution", "authors": "Thomas Vandal et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1703.03126", "role": "founding" }, { "title": "Residual Corrective Diffusion Modeling for Km-scale Atmospheric Downscaling", "authors": "Morteza Mardani et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2309.15214", "role": "state of the art" }, { "title": "ClimaX: A foundation model for weather and climate", "authors": "Tung Nguyen et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2301.10343", "role": "state of the art" } ], "approved": true }, { "id": "deep-learning-climate-emulation", "name": "Deep-learning climate emulation", "definition": "Learned models of the coupled Earth system that run for years to centuries, for seasonal and climate questions rather than daily weather.", "compute_anchor": "DLESyM: described as parsimonious and highly efficient; compute not stated in the abstract.", "data_anchor": "DLESyM: historical reanalysis plus satellite observations; stable 1000-year simulations without drift.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 7 authors on the DLESyM paper (arXiv 2409.16247).", "time_horizon": "Multi-year.", "shelf_life_signals": [ "DLESyM matches or exceeds four CMIP6 models on key variability metrics, including the Indian Summer monsoon." ], "exemplars": [ "dlesym", "nvidia-cbottle" ], "observed_actors": [ "university-of-washington", "nvidia", "google" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "University of Washington and NVIDIA publish long-run emulators.", "b": "Emulation is described as efficient, but no compute figure is given.", "c": "Running a released emulator is possible, but no compute figure is given.", "d": "Century-scale climate simulation is outside proof-of-concept scope." }, "tier_confidence": { "a": "medium", "b": "low", "c": "low", "d": "medium" }, "pipeline_role": "Tier A and universities build emulators; Tier B and Tier C study regional climate with them; Tier D has no observed role.", "sources": [ "https://arxiv.org/abs/2409.16247" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "WeatherBench: A benchmark dataset for data-driven weather forecasting", "authors": "Stephan Rasp et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2002.00469", "role": "founding" }, { "title": "ACE2: Accurately learning subseasonal to decadal atmospheric variability and forced responses", "authors": "Oliver Watt-Meyer et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2411.11268", "role": "state of the art" }, { "title": "LUCIE: A Lightweight Uncoupled ClImate Emulator with long-term stability and physical consistency for O(1000)-member ensembles", "authors": "Haiwen Guan et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2405.16297", "role": "state of the art" }, { "title": "ClimaX: A foundation model for weather and climate", "authors": "Tung Nguyen et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2301.10343", "role": "state of the art" }, { "title": "Spherical Fourier Neural Operators: Learning Stable Dynamics on the Sphere", "authors": "Boris Bonev et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.03838", "role": "state of the art" } ], "approved": true }, { "id": "heliophysics-foundation-models", "name": "Heliophysics foundation models", "definition": "Models pretrained on full-resolution solar observations to forecast solar flares, solar wind and other space weather.", "compute_anchor": "Surya: 366M parameters; fine-tuned with LoRA; training compute not stated in the abstract.", "data_anchor": "Surya: multi-instrument Solar Dynamics Observatory data, eight AIA channels and five HMI products.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 33 authors on the Surya paper (arXiv 2508.14112).", "time_horizon": "Years.", "shelf_life_signals": [ "Surya is described as the first heliophysics foundation model to use time advancement as a pretext task." ], "exemplars": [ "ibm-surya" ], "observed_actors": [ "ibm", "nasa" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "not feasible", "tier_rationale": { "a": "IBM and NASA built the 366M-parameter Surya.", "b": "LoRA fine-tuning of a 366M model is small, but space weather is not a shown Indonesian lab focus.", "c": "LoRA fine-tuning of a 366M model on public SDO data fits a university astronomy unit.", "d": "No delivery use case for space weather is in the use case list." }, "tier_confidence": { "a": "high", "b": "low", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A releases the model; Tier C fine-tunes it on public SDO data; Tier B and Tier D have no observed role.", "sources": [ "https://arxiv.org/abs/2508.14112" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "A Machine Learning Dataset Prepared From the NASA Solar Dynamics Observatory Mission", "authors": "Richard Galvez et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1903.04538", "role": "founding" }, { "title": "Surya: Foundation Model for Heliophysics", "authors": "Sujit Roy et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2508.14112", "role": "state of the art" } ], "approved": true }, { "id": "biomolecular-structure-and-co-folding", "name": "Biomolecular structure prediction and co-folding", "definition": "Predicting the 3D structure of proteins and their complexes with other proteins, DNA, RNA and drug molecules.", "compute_anchor": "Protenix-Mini cuts inference cost with a few-step ODE sampler, pruned blocks and an ESM module in place of the MSA; training compute not stated.", "data_anchor": "Boltz-2 jointly models complex structures and binding affinities; Boltz-1 is described as the first fully open model to approach AlphaFold3 accuracy.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on the Protenix-Mini paper (arXiv 2507.11839).", "time_horizon": "Years.", "shelf_life_signals": [ "Boltz-1 is described as the first fully open model to approach AlphaFold3 accuracy, and Boltz-2 adds binding affinity." ], "exemplars": [ "google-deepmind-alphafold-3", "boltz-boltz-2", "bytedance-protenix-v1", "chai-discovery-chai-1" ], "observed_actors": [ "google-deepmind", "isomorphic-labs", "boltz", "mit", "bytedance", "chai-discovery", "openfold-consortium", "microsoft" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Google DeepMind, ByteDance and the Boltz team train co-folding models.", "b": "Training is not sourced, but open Boltz and Protenix weights can be run and fine-tuned.", "c": "Running open Boltz or Protenix for local targets fits a university structural biology unit.", "d": "Inference on open weights supports a drug-discovery proof of concept, but needs a pharma partner with targets." }, "tier_confidence": { "a": "high", "b": "low", "c": "medium", "d": "low" }, "pipeline_role": "Tier A and startups release co-folding models; Tier C runs them on local targets; Tier D could support screening proofs of concept.", "sources": [ "https://arxiv.org/abs/2507.11839", "https://raw.githubusercontent.com/jwohlwend/boltz/main/README.md" ], "unverified": [ "compute_anchor" ], "approved": true }, { "id": "protein-language-models-and-generative-protein-design", "name": "Protein language models and generative protein design", "definition": "Language and diffusion models over protein sequences and structures that generate new proteins, binders and antibodies.", "compute_anchor": "ESM3: 98B parameters trained with over 1x10^24 FLOPS, which EvolutionaryScale calls the most compute applied to a biological model.", "data_anchor": "ESM3: a multimodal dataset augmented with hundreds of millions of synthetic predicted structures and functions.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): Not in the sources read (the ESM3 blog gives no author count).", "time_horizon": "Multi-year.", "shelf_life_signals": [ "ESM3 is a generative model over sequence, structure and function together, beyond sequence-only protein language models." ], "exemplars": [ "evolutionaryscale-esm3", "rfdiffusion", "proteinmpnn" ], "observed_actors": [ "evolutionaryscale", "meta", "microsoft", "boltz", "chai-discovery" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "EvolutionaryScale trained ESM3 with over 1x10^24 FLOPS.", "b": "Pretraining at 1x10^24 FLOPS is out of reach; use of open models is possible but no Indonesian actor is observed.", "c": "Running open RFdiffusion and ProteinMPNN fits a lab with wet-lab validation, which is not shown.", "d": "Protein design needs wet-lab testing outside delivery scope." }, "tier_confidence": { "a": "high", "b": "medium", "c": "low", "d": "medium" }, "pipeline_role": "Tier A trains protein models; Tier C designs and tests proteins with open ones; Tier B and Tier D have no observed role.", "sources": [ "https://www.evolutionaryscale.ai/blog/esm3-release", "https://raw.githubusercontent.com/RosettaCommons/RFdiffusion/main/README.md" ], "unverified": [ "team_size_proxy" ], "papers": [ { "title": "ProGen: Language Modeling for Protein Generation", "authors": "Ali Madani et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2004.03497", "role": "founding" }, { "title": "ProtTrans: Towards Cracking the Language of Life's Code Through Self-Supervised Deep Learning and High Performance Computing", "authors": "Ahmed Elnaggar et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2007.06225", "role": "founding" } ], "approved": true }, { "id": "genomic-dna-language-models", "name": "Genomic DNA language models", "definition": "Language models trained on DNA and RNA sequences to predict variant effects, detect pathogens and design sequences.", "compute_anchor": "Evo 2: 1B to 40B parameters with up to 1 million base pair context; METAGENE-1: 7B parameters.", "data_anchor": "Evo 2: OpenGenome2, 8.8 trillion tokens from all domains of life; METAGENE-1: over 1.5 trillion base pairs from wastewater.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 7 authors on the METAGENE-1 paper (arXiv 2501.02045).", "time_horizon": "Years.", "shelf_life_signals": [ "METAGENE-1 reports state-of-the-art results on genomic benchmarks shortly after Evo 2." ], "exemplars": [ "arc-institute-evo-2", "metagene-1" ], "observed_actors": [ "arc-institute", "nvidia", "usc", "prime-intellect", "google-deepmind", "google" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Arc Institute with NVIDIA trained Evo 2 on 8.8 trillion tokens.", "b": "Pretraining on 8.8 trillion tokens is out of reach, but the open 1B and 7B Evo 2 models can be run locally.", "c": "Running open Evo 2 7B or METAGENE-1 7B for variant or pathogen studies fits a university genomics unit.", "d": "Pathogen surveillance on open 7B models is a possible proof of technology with a health partner." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "low" }, "pipeline_role": "Tier A trains genomic models; Tier B and Tier C apply open ones to Indonesian genomes and surveillance; Tier D could support a health proof of technology.", "sources": [ "https://raw.githubusercontent.com/ArcInstitute/evo2/main/README.md", "https://arxiv.org/abs/2501.02045" ], "unverified": [], "papers": [ { "title": "HyenaDNA: Long-Range Genomic Sequence Modeling at Single Nucleotide Resolution", "authors": "Eric Nguyen et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2306.15794", "role": "state of the art" } ], "approved": true }, { "id": "single-cell-and-cell-imaging-foundation-models", "name": "Single-cell and cell imaging foundation models", "definition": "Models pretrained on gene expression of single cells or on cell microscopy images, used to annotate cells and predict perturbations.", "compute_anchor": "C2S-Scale: fine-tuned from Gemma 2 at 2B and 27B parameters; training compute not stated.", "data_anchor": "Geneformer V1: about 30 million single-cell transcriptomes, expanded model about 104 million; C2S-Scale: over 57 million human and mouse cells from over 800 datasets.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): Not in the sources read (model cards give no author count).", "time_horizon": "Years.", "shelf_life_signals": [ "C2S-Scale converts expression into text for a general LLM, an alternative to purpose-built models such as Geneformer." ], "exemplars": [ "geneformer", "c2s-scale" ], "observed_actors": [ "yale-university", "google", "google-deepmind", "recursion" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "not feasible", "tier_rationale": { "a": "Google with Yale fine-tuned Gemma 2 on over 57 million cells.", "b": "Open checkpoints exist, but no Indonesian single-cell programme is observed.", "c": "Running open Geneformer or C2S-Scale 2B on public atlases fits a university biology unit.", "d": "Single-cell analysis is a research activity without a delivery use case in the list." }, "tier_confidence": { "a": "high", "b": "low", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A releases cell models; Tier C applies them to local datasets; Tier B and Tier D have no observed role.", "sources": [ "https://huggingface.co/ctheodoris/Geneformer/raw/main/README.md", "https://huggingface.co/vandijklab/C2S-Scale-Gemma-2-27B/raw/main/README.md" ], "unverified": [ "compute_anchor", "team_size_proxy" ], "papers": [ { "title": "Cell Detection with Star-convex Polygons", "authors": "Uwe Schmidt et al.", "year": 2018, "venue": "MICCAI 2018 (per arXiv comment)", "url": "https://arxiv.org/abs/1806.03535", "role": "founding" } ], "approved": true }, { "id": "equivariant-graph-networks-for-molecules", "name": "Equivariant graph networks for molecules", "definition": "Graph neural networks that respect rotations and translations of atoms, used to predict molecular properties and dynamics.", "compute_anchor": "EGNN avoids costly higher-order representations; compute not stated in the abstract.", "data_anchor": "Molecular property benchmarks; SchNet models energies and forces of molecules.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on the EGNN paper (arXiv 2102.09844); 6 on SchNet.", "time_horizon": "Years.", "shelf_life_signals": [ "These architectures became the building blocks of universal interatomic potentials on Matbench Discovery." ], "exemplars": [ "egnn", "schnet" ], "observed_actors": [ "university-of-amsterdam", "tu-berlin", "tum", "microsoft-research", "university-of-oxford" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Microsoft Research and universities publish these architectures.", "b": "Small architectures on public benchmarks fit a modest budget.", "c": "Small architectures on public benchmarks fit a university chemistry or physics lab.", "d": "Property prediction for a formulation is possible but needs domain data." }, "tier_confidence": { "a": "medium", "b": "medium", "c": "high", "d": "low" }, "pipeline_role": "Universities publish architectures; Tier B and Tier C train them on local chemistry; Tier D could use them in formulation proofs.", "sources": [ "https://arxiv.org/abs/2102.09844", "https://arxiv.org/abs/1706.08566" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Tensor field networks: Rotation- and translation-equivariant neural networks for 3D point clouds", "authors": "Nathaniel Thomas et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1802.08219", "role": "founding" }, { "title": "Neural Message Passing for Quantum Chemistry", "authors": "Justin Gilmer et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1704.01212", "role": "founding" }, { "title": "MACE: Higher Order Equivariant Message Passing Neural Networks for Fast and Accurate Force Fields", "authors": "Ilyes Batatia et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2206.07697", "role": "state of the art" }, { "title": "E(n) Equivariant Graph Neural Networks", "authors": "Victor Garcia Satorras et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2102.09844", "role": "state of the art" }, { "title": "Geometric Deep Learning: Grids, Groups, Graphs, Geodesics, and Gauges", "authors": "Michael M. Bronstein et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2104.13478", "role": "survey" } ], "approved": true }, { "id": "universal-machine-learned-interatomic-potentials", "name": "Universal machine-learned interatomic potentials", "definition": "Force fields learned from first-principles calculations that simulate materials across the periodic table far faster than DFT.", "compute_anchor": "MatterSim: compute not stated in the abstract; Matbench Discovery ranks potentials by a combined score.", "data_anchor": "MatterSim: actively learned from large-scale first-principles computations covering 0 to 5000 K and up to 1000 GPa.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 22 authors on the MatterSim paper (arXiv 2405.04967); 10 on Matbench Discovery.", "time_horizon": "Months.", "shelf_life_signals": [ "Matbench Discovery keeps an online leaderboard for new submissions, so the best potential changes as entries arrive." ], "exemplars": [ "microsoft-mattersim", "meta-esen-30m-oam", "epfl-pet-oam-xl" ], "observed_actors": [ "microsoft", "meta", "epfl", "harvard-university", "orbital-materials", "seoul-national-university", "deepmodeling", "ict-cas", "icams-bochum", "massachusetts-institute-of-technology" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Microsoft and Meta train potentials on large first-principles datasets.", "b": "Training data comes from large DFT campaigns, but released potentials can be run locally.", "c": "Running a released potential for local materials is a cheap replacement for DFT in a university lab.", "d": "Screening candidate materials with a released potential is a possible proof of technology for chemical or mining clients." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "low" }, "pipeline_role": "Tier A releases universal potentials; Tier C runs them on local materials questions; Tier D could screen candidates in a proof of technology.", "sources": [ "https://arxiv.org/abs/2405.04967", "https://arxiv.org/abs/2308.14920" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "UMA: A Family of Universal Models for Atoms", "authors": "Brandon M. Wood et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2506.23971", "role": "state of the art" }, { "title": "A foundation model for atomistic materials chemistry", "authors": "Ilyes Batatia et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2401.00096", "role": "state of the art" }, { "title": "Matbench Discovery -- A framework to evaluate machine learning crystal stability predictions", "authors": "Janosh Riebesell et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2308.14920", "role": "state of the art" }, { "title": "A Universal Graph Deep Learning Interatomic Potential for the Periodic Table", "authors": "Chi Chen et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2202.02450", "role": "state of the art" } ], "approved": true }, { "id": "generative-crystal-materials-design", "name": "Generative design of crystalline materials", "definition": "Diffusion models that generate new stable crystal structures steered towards target chemistry and properties.", "compute_anchor": "MatterGen: compute not stated in the abstract.", "data_anchor": "MatterGen: fine-tuned with adapter modules on labelled property datasets; produces structures more than twice as likely to be novel and stable as prior models.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 21 authors on the MatterGen paper (arXiv 2312.03687).", "time_horizon": "Multi-year.", "shelf_life_signals": [ "MatterGen reports a design with low supply-chain-risk chemistry, pointing to applied use." ], "exemplars": [ "microsoft-mattergen" ], "observed_actors": [ "microsoft" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "Microsoft Research built MatterGen.", "b": "Adapter fine-tuning on labelled property data is possible, but compute is not stated.", "c": "Adapter fine-tuning is possible, but synthesis to confirm a design needs a materials lab.", "d": "Generated crystals need synthesis and testing outside delivery scope." }, "tier_confidence": { "a": "high", "b": "low", "c": "low", "d": "medium" }, "pipeline_role": "Tier A trains generative models; Tier C tests candidates; Tier B and Tier D have no observed role.", "sources": [ "https://arxiv.org/abs/2312.03687" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Crystal Diffusion Variational Autoencoder for Periodic Material Generation", "authors": "Tian Xie et al.", "year": 2021, "venue": "ICLR 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2110.06197", "role": "founding" }, { "title": "Fine-Tuned Language Models Generate Stable Inorganic Materials as Text", "authors": "Nate Gruver et al.", "year": 2024, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2402.04379", "role": "state of the art" }, { "title": "FlowMM: Generating Materials with Riemannian Flow Matching", "authors": "Benjamin Kurt Miller et al.", "year": 2024, "venue": "ICML 2024", "url": "https://arxiv.org/abs/2406.04713", "role": "state of the art" }, { "title": "Crystal Structure Prediction by Joint Equivariant Diffusion", "authors": "Rui Jiao et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2309.04475", "role": "state of the art" }, { "title": "MatterGen: a generative model for inorganic materials design", "authors": "Claudio Zeni et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.03687", "role": "state of the art" } ], "approved": true }, { "id": "latent-dynamics-of-neural-recordings", "name": "Latent dynamics models of neural population recordings", "definition": "Sequence models that infer low-dimensional dynamics from many simultaneously recorded neurons.", "compute_anchor": "LFADS: a variational auto-encoder; compute not stated in the abstract.", "data_anchor": "LFADS: single-trial, high-dimensional neural spiking data and synthetic data.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on the LFADS paper (arXiv 1608.06315).", "time_horizon": "Years.", "shelf_life_signals": [], "exemplars": [ "lfads" ], "observed_actors": [ "google", "stanford-university" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "Google with Stanford published LFADS.", "b": "The model is small, but no Indonesian neuroscience recording programme is observed.", "c": "The model is small, but it needs multi-electrode neural recordings.", "d": "Neural recording analysis has no delivery use case in the list." }, "tier_confidence": { "a": "medium", "b": "low", "c": "low", "d": "medium" }, "pipeline_role": "Tier A and universities publish methods; Tier C applies them where recordings exist; Tier B and Tier D have no observed role.", "sources": [ "https://arxiv.org/abs/1608.06315" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "A Unified, Scalable Framework for Neural Population Decoding", "authors": "Mehdi Azabou et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2310.16046", "role": "state of the art" }, { "title": "Neuroformer: Multimodal and Multitask Generative Pretraining for Brain Data", "authors": "Antonis Antoniades et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2311.00136", "role": "state of the art" }, { "title": "Representation learning for neural population activity with Neural Data Transformers", "authors": "Joel Ye et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2108.01210", "role": "state of the art" }, { "title": "Neural Latents Benchmark '21: Evaluating latent variable models of neural population activity", "authors": "Felix Pei et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2109.04463", "role": "state of the art" } ], "approved": true }, { "id": "deep-reinforcement-learning-for-control", "name": "Deep reinforcement learning algorithms for control", "definition": "General algorithms that learn control policies from reward, online or from fixed logged data.", "compute_anchor": "IMPALA scales to thousands of machines; PPO allows multiple epochs of minibatch updates per sample; compute per task not stated.", "data_anchor": "IMPALA: DMLab-30 and Atari-57; CQL: fixed offline datasets without further interaction.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on the PPO paper (arXiv 1707.06347); 12 on IMPALA.", "time_horizon": "Years.", "shelf_life_signals": [ "PPO remains the default policy-gradient method years after 2017; offline RL (CQL) grew for settings where interaction is costly." ], "exemplars": [ "ppo", "sac", "cql" ], "observed_actors": [ "openai", "uc-berkeley", "google-deepmind", "mcgill-university", "google" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "OpenAI and Google DeepMind publish the core algorithms and scale IMPALA to thousands of machines.", "b": "PPO and SAC run on simulated robotics benchmarks with modest compute.", "c": "PPO and SAC are standard teaching and research baselines in university labs.", "d": "Offline RL with CQL learns from logged data without interaction, which fits a process-optimisation proof of concept." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "medium" }, "pipeline_role": "Tier A publishes algorithms; Tier B and Tier C apply them in simulation; Tier D uses offline RL on client logs.", "sources": [ "https://arxiv.org/abs/1707.06347", "https://arxiv.org/abs/1801.01290", "https://arxiv.org/abs/2006.04779", "https://arxiv.org/abs/1802.01561" ], "unverified": [], "papers": [ { "title": "Playing Atari with Deep Reinforcement Learning", "authors": "Volodymyr Mnih et al.", "year": 2013, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1312.5602", "role": "founding" }, { "title": "Continuous control with deep reinforcement learning", "authors": "Timothy P. Lillicrap et al.", "year": 2015, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1509.02971", "role": "founding" }, { "title": "Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor", "authors": "Tuomas Haarnoja et al.", "year": 2018, "venue": "ICML 2018 (per arXiv comment)", "url": "https://arxiv.org/abs/1801.01290", "role": "state of the art" }, { "title": "Proximal Policy Optimization Algorithms", "authors": "John Schulman et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1707.06347", "role": "state of the art" }, { "title": "Deep Reinforcement Learning: An Overview", "authors": "Yuxi Li", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1701.07274", "role": "survey" } ], "approved": true }, { "id": "self-play-reinforcement-learning-with-search", "name": "Self-play reinforcement learning with search", "definition": "Agents that reach superhuman play by playing against themselves at scale, often with tree search.", "compute_anchor": "OpenAI Five: batches of about 2 million frames every 2 seconds, trained for 10 months.", "data_anchor": "OpenAI Five: self-play in Dota 2; MuZero: 57 Atari games plus Go, chess and shogi without given rules.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 27 authors on the OpenAI Five paper (arXiv 1912.06680); 12 on MuZero.", "time_horizon": "Multi-year.", "shelf_life_signals": [ "MuZero dropped the need for known rules that AlphaZero had." ], "exemplars": [ "google-deepmind-alphazero", "google-deepmind-muzero", "openai-five" ], "observed_actors": [ "google-deepmind", "openai" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "OpenAI trained Five for 10 months on about 2 million frames every 2 seconds.", "b": "Ten months of distributed training is out of reach on the evidence.", "c": "Small-scale reproductions on board games are possible, but frontier-scale runs are not.", "d": "No delivery use is shown." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A runs self-play at scale; Tier C studies small reproductions; Tier B and Tier D have no observed role.", "sources": [ "https://arxiv.org/abs/1912.06680", "https://arxiv.org/abs/1911.08265" ], "unverified": [], "papers": [ { "title": "Student of Games: A unified learning algorithm for both perfect and imperfect information games", "authors": "Martin Schmid et al.", "year": 2021, "venue": "Science Advances 9, eadg3256 (2023)", "url": "https://arxiv.org/abs/2112.03178", "role": "state of the art" }, { "title": "Mastering Atari, Go, Chess and Shogi by Planning with a Learned Model", "authors": "Julian Schrittwieser et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1911.08265", "role": "state of the art" }, { "title": "Mastering Chess and Shogi by Self-Play with a General Reinforcement Learning Algorithm", "authors": "David Silver et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1712.01815", "role": "state of the art" } ], "approved": true }, { "id": "reinforcement-learning-for-algorithm-discovery", "name": "Reinforcement learning for algorithm discovery", "definition": "Using game-playing reinforcement learning agents to find new, provably correct algorithms, such as faster matrix multiplication or sorting.", "compute_anchor": "AlphaTensor: compute not stated in the source.", "data_anchor": "AlphaTensor: matrix multiplication posed as a single-player game, building on AlphaZero.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): Not in the sources read (the blog gives no author count).", "time_horizon": "Multi-year.", "shelf_life_signals": [ "AlphaTensor addressed a 50-year-old open question since Strassen in 1969." ], "exemplars": [ "google-deepmind-alphatensor", "google-deepmind-alphadev" ], "observed_actors": [ "google-deepmind" ], "tier_a": "feasible", "tier_b": "not feasible", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "Google DeepMind built AlphaTensor and AlphaDev on AlphaZero.", "b": "Compute is not stated and the actor is frontier only.", "c": "Mathematics groups could study the published algorithms, but compute is not stated.", "d": "No delivery use is shown." }, "tier_confidence": { "a": "high", "b": "low", "c": "low", "d": "medium" }, "pipeline_role": "Tier A discovers algorithms; Tier C studies and verifies them; Tier B and Tier D have no observed role.", "sources": [ "https://deepmind.google/discover/blog/discovering-novel-algorithms-with-alphatensor/" ], "unverified": [ "compute_anchor", "team_size_proxy" ], "papers": [ { "title": "Discovering Reinforcement Learning Algorithms", "authors": "Junhyuk Oh et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2007.08794", "role": "founding" }, { "title": "Eureka: Human-Level Reward Design via Coding Large Language Models", "authors": "Yecheng Jason Ma et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2310.12931", "role": "state of the art" } ], "approved": true }, { "id": "text-embeddings-for-semantic-search", "name": "Text embeddings for semantic search", "definition": "Training encoders that turn text in many languages into vectors so that search, clustering and retrieval-augmented generation match meaning rather than keywords.", "compute_anchor": "Qwen3 Embedding: embedding models of 0.6B, 4B and 8B parameters built on Qwen3 LLM backbones; training compute is not given in the abstract.", "data_anchor": "Qwen3 Embedding: multi-stage training of large-scale unsupervised pretraining then supervised fine-tuning, with Qwen3 LLMs synthesising training data across domains and languages. C-Pack: C-MTP training set and the C-MTEB benchmark of 6 tasks and 35 datasets.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 12 authors on the Qwen3 Embedding paper; 6 on C-Pack.", "time_horizon": "Months.", "shelf_life_signals": [ "MTEB leadership moving from BERT-size encoders to LLM-backbone embedders (Qwen3 Embedding reports top multilingual MTEB results)", "Adjustable vector sizes (32 to 4,096 dimensions on Qwen3 Embedding) cutting index storage" ], "exemplars": [ "alibaba-qwen3-embedding-8b", "baai-bge-m3", "alibaba-gte-multilingual-base", "microsoft-multilingual-e5-large-instruct", "ai-singapore-sea-lion-e5-embedding-600m" ], "observed_actors": [ "alibaba", "baai", "google", "jina-ai", "nvidia", "nomic-ai", "sentence-transformers", "cohere" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Alibaba and BAAI train embedders of 0.6B to 8B on LLM backbones with LLM-synthesised data (Qwen3 Embedding, C-Pack).", "b": "A 0.6B embedder fine-tuned from an open backbone, as in Qwen3 Embedding, is within reach for Indonesian retrieval data; Indonesian lab capacity itself is not in the sources.", "c": "C-Pack shows a 6-author team building a benchmark of 35 datasets plus training data, a scale an institution unit can match for Indonesian.", "d": "Open Apache 2.0 embedders (Qwen3 Embedding) can be used as they are in search and RAG proofs of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A releases multilingual embedders and benchmarks; Tier B and Tier C fine-tune small ones on Indonesian and domain text and build local test sets; Tier D puts them into search and RAG proofs of value.", "sources": [ "https://arxiv.org/abs/2506.05176", "https://arxiv.org/abs/2309.07597" ], "unverified": [ "compute_anchor (training compute)" ], "papers": [ { "title": "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks", "authors": "Nils Reimers et al.", "year": 2019, "venue": "EMNLP 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1908.10084", "role": "founding" }, { "title": "Dense Passage Retrieval for Open-Domain Question Answering", "authors": "Vladimir Karpukhin et al.", "year": 2020, "venue": "EMNLP 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2004.04906", "role": "founding" }, { "title": "Qwen3 Embedding: Advancing Text Embedding and Reranking Through Foundation Models", "authors": "Yanzhao Zhang et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2506.05176", "role": "state of the art" }, { "title": "C-Pack: Packed Resources For General Chinese Embeddings", "authors": "Shitao Xiao et al.", "year": 2023, "venue": "SIGIR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2309.07597", "role": "state of the art" }, { "title": "When Text Embedding Meets Large Language Model: A Comprehensive Survey", "authors": "Zhijie Nie et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2412.09165", "role": "survey" } ], "approved": true }, { "id": "neural-rerankers-for-retrieval", "name": "Neural rerankers for retrieval", "definition": "Cross-encoder or LLM-based models that score each query and candidate document together to reorder a first-stage retrieval list.", "compute_anchor": "Qwen3 Embedding series ships rerankers of 0.6B, 4B and 8B parameters on Qwen3 backbones; training compute is not given in the abstract.", "data_anchor": "BGE (C-Pack) rerankers are trained alongside the C-MTP data and evaluated on C-MTEB (35 datasets); Qwen3 rerankers use the same multi-stage pipeline with LLM-synthesised data.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 12 authors on the Qwen3 Embedding paper; 6 on C-Pack.", "time_horizon": "Months.", "shelf_life_signals": [ "Rerankers moving from small cross-encoders (MiniLM) to LLM backbones (Qwen3, Llama Nemotron, mxbai on Qwen-2.5)", "Multimodal rerankers that score page images (Qwen3-VL-Reranker, Llama Nemotron Rerank VL)" ], "exemplars": [ "alibaba-qwen3-reranker-0-6b", "baai-bge-reranker-v2-m3", "sentence-transformers-ms-marco-minilm-l6-v2", "cohere-rerank-v3-5" ], "observed_actors": [ "alibaba", "baai", "cohere", "sentence-transformers", "jina-ai", "nvidia", "voyage-ai", "mixedbread" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Alibaba, BAAI, NVIDIA and Cohere release rerankers from 0.6B to 8B (Qwen3 Embedding, C-Pack).", "b": "Fine-tuning a 0.6B reranker on Indonesian query and passage pairs follows the published Qwen3 and BGE recipes; Indonesian lab capacity is not in the sources.", "c": "Reranker training needs pairs and a benchmark of the C-MTEB kind, which a 6-author team built for Chinese (C-Pack).", "d": "A reranker added after vector search is a cheap quality step in RAG proofs of concept with open 0.6B models (Qwen3 Embedding)." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A releases rerankers; Tier C builds Indonesian relevance sets and fine-tunes small rerankers; Tier D adds them to retrieval pipelines in proofs of value.", "sources": [ "https://arxiv.org/abs/2506.05176", "https://arxiv.org/abs/2309.07597" ], "unverified": [ "compute_anchor (training compute)" ], "papers": [ { "title": "Passage Re-ranking with BERT", "authors": "Rodrigo Nogueira et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1901.04085", "role": "founding" }, { "title": "Is ChatGPT Good at Search? Investigating Large Language Models as Re-Ranking Agents", "authors": "Weiwei Sun et al.", "year": 2023, "venue": "EMNLP 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2304.09542", "role": "state of the art" }, { "title": "RankZephyr: Effective and Robust Zero-Shot Listwise Reranking is a Breeze!", "authors": "Ronak Pradeep et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.02724", "role": "state of the art" }, { "title": "ColBERT: Efficient and Effective Passage Search via Contextualized Late Interaction over BERT", "authors": "Omar Khattab et al.", "year": 2020, "venue": "SIGIR 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2004.12832", "role": "state of the art" }, { "title": "Pretrained Transformers for Text Ranking: BERT and Beyond", "authors": "Jimmy Lin et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2010.06467", "role": "survey" } ], "approved": true }, { "id": "visual-document-retrieval", "name": "Visual document retrieval", "definition": "Retrieving document pages by embedding the page images themselves with a vision language model, instead of extracting text first.", "compute_anchor": "ColPali is a vision language model producing multi-vector embeddings from page images; parameter count and compute are not in the abstract.", "data_anchor": "ViDoRe benchmark of page-level retrieval tasks across domains, languages and settings, released with ColPali.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 7 authors on ColPali.", "time_horizon": "Months.", "shelf_life_signals": [ "Late-interaction multi-vector retrieval (ColPali) versus single-vector multimodal embedders (Qwen3-VL-Embedding, Cohere Embed v4)", "RAG over scanned and visually rich documents without an OCR step" ], "exemplars": [ "illuin-colpali", "illuin-colqwen2-v1-0", "nvidia-colembed-v2", "jina-ai-jina-embeddings-v4" ], "observed_actors": [ "alibaba", "cohere", "jina-ai", "nvidia", "illuin-technology", "baai", "nomic-ai", "google" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google, NVIDIA, Alibaba and Cohere ship multimodal embedders that index page images.", "b": "ColPali is trained from an open VLM with a benchmark built by a 7-author team, a scale an Indonesian lab could follow; local capacity is not in the sources.", "c": "Building an Indonesian ViDoRe-style test set and fine-tuning ColPali is within an institution unit's reach (7-author paper).", "d": "ColPali is released with open models, data and code, so scanned-document RAG proofs of concept can use it directly." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A and open labs release page-image embedders; Tier C builds Indonesian document test sets; Tier D applies them to RAG over scanned forms, reports and manuals.", "sources": [ "https://arxiv.org/abs/2407.01449" ], "unverified": [ "compute_anchor (parameters, training compute)" ], "papers": [ { "title": "Unifying Multimodal Retrieval via Document Screenshot Embedding", "authors": "Xueguang Ma et al.", "year": 2024, "venue": "EMNLP2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2406.11251", "role": "founding" }, { "title": "ColPali: Efficient Document Retrieval with Vision Language Models", "authors": "Manuel Faysse et al.", "year": 2024, "venue": "ICLR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2407.01449", "role": "state of the art" }, { "title": "VisRAG: Vision-based Retrieval-augmented Generation on Multi-modality Documents", "authors": "Shi Yu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2410.10594", "role": "state of the art" }, { "title": "M3DocRAG: Multi-modal Retrieval is What You Need for Multi-page Multi-document Understanding", "authors": "Jaemin Cho et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2411.04952", "role": "state of the art" } ], "approved": true }, { "id": "pretrained-bidirectional-text-encoders", "name": "Pretrained bidirectional text encoders", "definition": "Pretraining BERT-style encoder-only models that read text in both directions, used for classification, entity extraction and as the base of embedders.", "compute_anchor": "ModernBERT: trained on 2 trillion tokens with a native 8,192-token sequence length; designed for inference on common GPUs.", "data_anchor": "IndoBERT: trained on Indo4B, a large clean Indonesian dataset from social media, blogs, news and websites, with the 12-task IndoNLU benchmark.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 14 authors on ModernBERT; 11 on IndoNLU.", "time_horizon": "One to two years.", "shelf_life_signals": [ "Encoder refreshes with long context (ModernBERT 8,192 tokens) and regional retrains (SEA-LION ModernBERT)", "Encoders kept in production for their performance-size trade-off against decoder LLMs" ], "exemplars": [ "modernbert", "indonlp-indobert", "microsoft-mdeberta-v3-base", "ai-singapore-sea-lion-modernbert-600m" ], "observed_actors": [ "ai-singapore", "indonlp", "liquid-ai", "microsoft", "answer-ai", "lighton", "moritzlaurer", "allen-ai" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Encoder pretraining at 2 trillion tokens (ModernBERT) is small next to frontier LLM runs.", "b": "IndoBERT shows Indonesian encoders trained on Indo4B with a 12-task benchmark (IndoNLU).", "c": "IndoNLU was produced by an 11-author academic and industry team, the scale of an institution unit.", "d": "Encoders such as IndoBERT run on common GPUs for classification and extraction in proofs of concept (ModernBERT targets common GPUs)." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier B and Tier C pretrain Indonesian encoders and benchmarks; Tier D fine-tunes them for classification and extraction in proofs of value; Tier A supplies modern recipes.", "sources": [ "https://arxiv.org/abs/2412.13663", "https://arxiv.org/abs/2009.05387" ], "unverified": [ "compute_anchor (training compute)" ], "papers": [ { "title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding", "authors": "Jacob Devlin et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1810.04805", "role": "founding" }, { "title": "NeoBERT: A Next-Generation BERT", "authors": "Lola Le Breton et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2502.19587", "role": "state of the art" }, { "title": "Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference", "authors": "Benjamin Warner et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2412.13663", "role": "state of the art" }, { "title": "ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators", "authors": "Kevin Clark et al.", "year": 2020, "venue": "ICLR 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2003.10555", "role": "state of the art" }, { "title": "DeBERTa: Decoding-enhanced BERT with Disentangled Attention", "authors": "Pengcheng He et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2006.03654", "role": "state of the art" } ], "approved": true }, { "id": "generative-recommendation", "name": "Generative recommendation", "definition": "Recasting recommendation as sequence generation over user action histories with large transformer models that scale with training compute.", "compute_anchor": "HSTU-based Generative Recommenders reach 1.5 trillion parameters, and quality scales as a power law of training compute across three orders of magnitude up to GPT-3 or Llama-2 scale.", "data_anchor": "Industrial streams of tens of billions of user actions a day (HSTU); OneRec encodes user behaviour sequences and decodes videos with sparse MoE and DPO-based preference alignment.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 12 authors on HSTU; 8 on OneRec.", "time_horizon": "One to two years.", "shelf_life_signals": [ "Deployed replacements of retrieve-and-rank cascades (OneRec at Kuaishou, HSTU on a platform with billions of users)", "Open LLM-backbone recommenders (OpenOneRec 1.7B and 8B)" ], "exemplars": [ "meta-hstu", "kuaishou-onerec", "kuaishou-onerec-v2", "kuaishou-openonerec" ], "observed_actors": [ "kuaishou", "meituan", "meta" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "stretch", "tier_rationale": { "a": "Meta and Kuaishou train and deploy generative recommenders up to 1.5 trillion parameters (HSTU).", "b": "Scaling gains need tens of billions of daily actions (HSTU), data held by platforms rather than labs; Indonesian lab access is not in the sources.", "c": "Open OneRec-Foundation 1.7B and 8B checkpoints allow study, but the scaling result needs industrial data (HSTU).", "d": "Proofs of value need a client's interaction logs at scale; small clients will not reach the HSTU regime." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A platforms publish architectures and open checkpoints; Tier C studies them on public data; Tier D applies smaller versions with clients who own large interaction logs.", "sources": [ "https://arxiv.org/abs/2402.17152", "https://arxiv.org/abs/2502.18965" ], "unverified": [ "compute_anchor (FLOP figure)" ], "papers": [ { "title": "Recommendation as Language Processing (RLP): A Unified Pretrain, Personalized Prompt & Predict Paradigm (P5)", "authors": "Shijie Geng et al.", "year": 2022, "venue": "RecSys 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2203.13366", "role": "founding" }, { "title": "OneRec: Unifying Retrieve and Rank with Generative Recommender and Iterative Preference Alignment", "authors": "Jiaxin Deng et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2502.18965", "role": "state of the art" }, { "title": "Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations", "authors": "Jiaqi Zhai et al.", "year": 2024, "venue": "ICML'24 (per arXiv comment)", "url": "https://arxiv.org/abs/2402.17152", "role": "state of the art" }, { "title": "Recommender Systems with Generative Retrieval", "authors": "Shashank Rajput et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2305.05065", "role": "state of the art" }, { "title": "Large Language Models for Generative Recommendation: A Survey and Visionary Discussions", "authors": "Lei Li et al.", "year": 2023, "venue": "COLING 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2309.01157", "role": "survey" } ], "approved": true }, { "id": "click-through-rate-and-sequential-recommendation", "name": "Click-through-rate prediction and sequential recommendation", "definition": "Neural models that predict clicks from sparse user and item features or predict the next item from a user's recent history.", "compute_anchor": "DIN and SASRec abstracts give no compute; SASRec is a self-attention model sized for single-machine training on public datasets (not stated numerically).", "data_anchor": "DIN: deployed on Alibaba online display advertising data; SASRec: user action histories from public datasets, sparse and dense.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 10 authors on DIN; 2 on SASRec.", "time_horizon": "Years (mature direction).", "shelf_life_signals": [ "Generative recommenders (HSTU, OneRec) displacing feature-interaction models at the largest platforms", "Graph collaborative filtering (LightGCN) as a standard baseline" ], "exemplars": [ "alibaba-din", "sasrec", "google-dcn-v2", "meta-dlrm" ], "observed_actors": [ "google", "university-of-science-and-technology-of-china", "alibaba", "alibaba-group", "nus", "harbin-institute-of-technology", "huawei-technologies", "microsoft-research" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google, Meta and Alibaba built and deployed these models (DIN in Alibaba display advertising).", "b": "Models of the SASRec kind are published with code and need modest compute.", "c": "SASRec was produced by a 2-author team, well within an institution unit.", "d": "CTR and next-item models fit e-commerce and media proofs of value with a client's own logs." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A publishes and deploys architectures; Tier C benchmarks them on local data; Tier D builds recommenders for clients in proofs of value.", "sources": [ "https://arxiv.org/abs/1706.06978", "https://arxiv.org/abs/1808.09781" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Session-based Recommendations with Recurrent Neural Networks", "authors": "Balázs Hidasi et al.", "year": 2015, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1511.06939", "role": "founding" }, { "title": "DCN V2: Improved Deep & Cross Network and Practical Lessons for Web-scale Learning to Rank Systems", "authors": "Ruoxi Wang et al.", "year": 2020, "venue": "In Proceedings of the Web Conference 2021 (WWW '21)", "url": "https://arxiv.org/abs/2008.13535", "role": "state of the art" }, { "title": "BERT4Rec: Sequential Recommendation with Bidirectional Encoder Representations from Transformer", "authors": "Fei Sun et al.", "year": 2019, "venue": "CIKM 2019 (per arXiv comment)", "url": "https://arxiv.org/abs/1904.06690", "role": "state of the art" }, { "title": "Deep Learning Recommendation Model for Personalization and Recommendation Systems", "authors": "Maxim Naumov et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1906.00091", "role": "state of the art" }, { "title": "Deep Learning based Recommender System: A Survey and New Perspectives", "authors": "Shuai Zhang et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1707.07435", "role": "survey" } ], "approved": true }, { "id": "graph-neural-networks-and-graph-transformers", "name": "Graph neural networks and graph transformers", "definition": "Neural networks that learn from nodes, edges and their features for node classification, link prediction and whole-graph property prediction.", "compute_anchor": "GCN scales linearly in the number of graph edges; GraphGPS gives a graph transformer with complexity linear in nodes and edges, O(N+E).", "data_anchor": "GCN: citation networks and a knowledge graph dataset; GraphGPS: a diverse set of graph benchmarks.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 2 authors on GCN; 6 on GraphGPS.", "time_horizon": "Years.", "shelf_life_signals": [ "Graph transformers (GraphGPS, Graphormer) moving past message passing on benchmarks", "Domain graph models such as GridSFM for power grids" ], "exemplars": [ "gcn", "graphsage", "graphgps", "microsoft-graphormer" ], "observed_actors": [ "stanford-university", "microsoft", "university-of-amsterdam", "mila", "university-of-cambridge", "technion", "massachusetts-institute-of-technology", "stony-brook-university" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Microsoft, Stanford and Mila publish graph architectures (GraphGPS, Graphormer).", "b": "Linear-complexity models (GCN, GraphGPS) need modest compute.", "c": "GCN came from a 2-author team, a scale any institution unit can match.", "d": "Fraud rings, supply networks and grids are graph problems a delivery team can prototype with GCN-class models; client graph data is the limit." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and universities publish architectures; Tier C applies them to Indonesian networks; Tier D prototypes fraud, logistics and grid use cases.", "sources": [ "https://arxiv.org/abs/1609.02907", "https://arxiv.org/abs/2205.12454" ], "unverified": [ "compute_anchor (training compute)" ], "papers": [ { "title": "Semi-Supervised Classification with Graph Convolutional Networks", "authors": "Thomas N. Kipf et al.", "year": 2016, "venue": "ICLR 2017 (per arXiv comment)", "url": "https://arxiv.org/abs/1609.02907", "role": "founding" }, { "title": "Recipe for a General, Powerful, Scalable Graph Transformer", "authors": "Ladislav Rampášek et al.", "year": 2022, "venue": "NeurIPS 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2205.12454", "role": "state of the art" }, { "title": "Do Transformers Really Perform Bad for Graph Representation?", "authors": "Chengxuan Ying et al.", "year": 2021, "venue": "NeurIPS 2021", "url": "https://arxiv.org/abs/2106.05234", "role": "state of the art" }, { "title": "How Powerful are Graph Neural Networks?", "authors": "Keyulu Xu et al.", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1810.00826", "role": "state of the art" }, { "title": "A Comprehensive Survey on Graph Neural Networks", "authors": "Zonghan Wu et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1901.00596", "role": "survey" } ], "approved": true }, { "id": "knowledge-graph-reasoning-and-relational-foundation-models", "name": "Knowledge graph reasoning and relational foundation models", "definition": "Models that predict missing links and entities in multi-relational graphs or relational databases, including pretrained models that transfer to unseen graphs.", "compute_anchor": "ULTRA: a single pre-trained model; parameter count and compute are not in the abstract.", "data_anchor": "ULTRA: link prediction experiments on 57 different knowledge graphs, zero-shot on unseen graphs.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on ULTRA.", "time_horizon": "One to two years.", "shelf_life_signals": [ "Zero-shot transfer to unseen graphs (ULTRA on 57 KGs)", "Relational foundation models over enterprise databases (KumoRFM)" ], "exemplars": [ "ultra-kg", "rotate-kge", "r-gcn" ], "observed_actors": [ "mila", "kumo-ai", "university-of-amsterdam", "intel", "peking-university" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Intel and Mila pretrain ULTRA; Kumo AI builds a relational foundation model.", "b": "One pre-trained ULTRA model covers 57 graphs, so pretraining compute is modest though not stated.", "c": "ULTRA came from a 5-author team, the size of an institution unit.", "d": "A pre-trained ULTRA model links entities zero-shot on a client's knowledge graph in a proof of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and universities release pretrained graph reasoners; Tier C builds Indonesian knowledge graphs; Tier D uses zero-shot link prediction on client data.", "sources": [ "https://arxiv.org/abs/2310.04562" ], "unverified": [ "compute_anchor (parameters, training compute)" ], "papers": [ { "title": "Modeling Relational Data with Graph Convolutional Networks", "authors": "Michael Schlichtkrull et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1703.06103", "role": "founding" }, { "title": "RelBench: A Benchmark for Deep Learning on Relational Databases", "authors": "Joshua Robinson et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2407.20060", "role": "state of the art" }, { "title": "Towards Foundation Models for Knowledge Graph Reasoning", "authors": "Mikhail Galkin et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2310.04562", "role": "state of the art" }, { "title": "Relational Deep Learning: Graph Representation Learning on Relational Databases", "authors": "Matthias Fey et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.04615", "role": "state of the art" }, { "title": "A Survey on Knowledge Graphs: Representation, Acquisition and Applications", "authors": "Shaoxiong Ji et al.", "year": 2020, "venue": "IEEE Transactions on Neural Networks and Learning Systems, 2021", "url": "https://arxiv.org/abs/2002.00388", "role": "survey" } ], "approved": true }, { "id": "deep-learning-and-gradient-boosting-for-tabular-data", "name": "Deep learning and gradient boosting for tabular data", "definition": "Comparing and improving tree ensembles and neural networks for prediction on rows and columns of business data.", "compute_anchor": "No compute figure; the benchmark accounts for hyperparameter search budgets across 45 datasets.", "data_anchor": "Standard set of 45 tabular datasets from varied domains; tree-based models remain state of the art on medium-sized data of about 10K samples.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on the tabular benchmark paper.", "time_horizon": "Years (mature direction).", "shelf_life_signals": [ "Tabular foundation models (TabPFN, TabICL) ranking near tuned boosting on TabArena", "Interpretable boosted models (EBM) for regulated decisions" ], "exemplars": [ "dmlc-xgboost", "yandex-catboost", "microsoft-lightgbm", "yandex-ft-transformer" ], "observed_actors": [ "yandex", "microsoft", "amazon", "dmlc", "inria", "peking-university" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Microsoft, Yandex and Amazon release the libraries and AutoML systems.", "b": "Benchmarks over 45 datasets need modest compute.", "c": "The benchmark paper came from a 3-author team.", "d": "Tree ensembles remain state of the art on about 10K-sample tables, the usual size of a client dataset in a proof of value." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and libraries supply tools; Tier C benchmarks them on Indonesian data; Tier D delivers most tabular proofs of value with them.", "sources": [ "https://arxiv.org/abs/2207.08815" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "XGBoost: A Scalable Tree Boosting System", "authors": "Tianqi Chen et al.", "year": 2016, "venue": "KDD'16 (per arXiv comment)", "url": "https://arxiv.org/abs/1603.02754", "role": "founding" }, { "title": "TabM: Advancing Tabular Deep Learning with Parameter-Efficient Ensembling", "authors": "Yury Gorishniy et al.", "year": 2024, "venue": "ICLR 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2410.24210", "role": "state of the art" }, { "title": "TabR: Tabular Deep Learning Meets Nearest Neighbors in 2023", "authors": "Yury Gorishniy et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2307.14338", "role": "state of the art" }, { "title": "Why do tree-based models still outperform deep learning on tabular data?", "authors": "Léo Grinsztajn et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2207.08815", "role": "state of the art" }, { "title": "Tabular Data: Deep Learning is Not All You Need", "authors": "Ravid Shwartz-Ziv et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2106.03253", "role": "survey" } ], "approved": true }, { "id": "table-question-answering", "name": "Table question answering", "definition": "Answering natural language questions over tables by selecting cells and aggregations or by generating SQL-like reasoning.", "compute_anchor": "TAPAS extends BERT to encode tables; size and compute are not in the abstract.", "data_anchor": "TAPAS: joint pretraining on text segments and tables crawled from Wikipedia, tested on three semantic parsing datasets including SQA.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 5 authors on TAPAS.", "time_horizon": "Months to a year.", "shelf_life_signals": [ "General LLMs with tool use answering table questions through SQL", "Table pretraining recipes (TAPAS, TAPEX) used as baselines" ], "exemplars": [ "google-tapas", "microsoft-tapex" ], "observed_actors": [ "google", "microsoft" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google and Microsoft published TAPAS and TAPEX.", "b": "BERT-size table models are small; Indonesian table data is not in the sources.", "c": "TAPAS came from a 5-author team.", "d": "Questions over client spreadsheets are a common proof-of-concept request; BERT-size TAPAS checkpoints are released." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A publishes table pretraining; Tier C builds Indonesian table question sets; Tier D answers questions over client tables.", "sources": [ "https://arxiv.org/abs/2004.02349" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Compositional Semantic Parsing on Semi-Structured Tables", "authors": "Panupong Pasupat et al.", "year": 2015, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1508.00305", "role": "founding" }, { "title": "Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task", "authors": "Tao Yu et al.", "year": 2018, "venue": "EMNLP 2018 (per arXiv comment)", "url": "https://arxiv.org/abs/1809.08887", "role": "founding" }, { "title": "Can LLM Already Serve as A Database Interface? A BIg Bench for Large-Scale Database Grounded Text-to-SQLs", "authors": "Jinyang Li et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2305.03111", "role": "state of the art" }, { "title": "TableLlama: Towards Open Large Generalist Models for Tables", "authors": "Tianshu Zhang et al.", "year": 2023, "venue": "NAACL 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2311.09206", "role": "state of the art" }, { "title": "A Survey of Table Reasoning with Large Language Models", "authors": "Xuanliang Zhang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.08259", "role": "survey" } ], "approved": true }, { "id": "fraud-and-outlier-detection-on-transaction-data", "name": "Fraud and outlier detection on transaction data", "definition": "Finding rare suspicious records such as fraudulent payments or money laundering in tabular transaction data, with supervised or unsupervised detectors.", "compute_anchor": "ECOD is parameter-free and estimates per-dimension empirical distributions, with no training compute to speak of.", "data_anchor": "HSBC with Google Cloud AML AI: two to four times as much suspicious activity found as the previous system, with 60% fewer alerts.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on ECOD.", "time_horizon": "Years.", "shelf_life_signals": [ "Bank-deployed AML models reporting fewer alerts (HSBC, 60%)", "Transaction embeddings from large behavioural models (Featurespace TallierLTM)" ], "exemplars": [ "google-aml-ai", "ecod", "pyod" ], "observed_actors": [ "carnegie-mellon-university", "northeastern-university", "featurespace", "google", "hsbc", "mastercard", "nudt" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google, Mastercard and Featurespace deploy fraud and AML models at bank scale (HSBC AML AI).", "b": "Unsupervised detectors such as ECOD need no labels or GPUs; access to bank data is not in the sources.", "c": "ECOD came from a 6-author team with a parameter-free method.", "d": "Parameter-free detectors (ECOD) give a fast first proof of concept on client transactions; labels and data access decide the next step." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "medium" }, "pipeline_role": "Tier A and payment networks deploy production models; Tier C benchmarks detectors on Indonesian data; Tier D runs unsupervised detection proofs of value with banks and fintechs.", "sources": [ "https://arxiv.org/abs/2201.00382", "https://cloud.google.com/blog/topics/financial-services/how-hsbc-fights-money-launderers-with-artificial-intelligence" ], "unverified": [], "papers": [ { "title": "Deep Semi-Supervised Anomaly Detection", "authors": "Lukas Ruff et al.", "year": 2019, "venue": "ICLR 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/1906.02694", "role": "founding" }, { "title": "Realistic Synthetic Financial Transactions for Anti-Money Laundering Models", "authors": "Erik Altman et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.16424", "role": "state of the art" }, { "title": "ADBench: Anomaly Detection Benchmark", "authors": "Songqiao Han et al.", "year": 2022, "venue": "NeurIPS 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2206.09426", "role": "state of the art" }, { "title": "Enhancing Graph Neural Network-based Fraud Detectors against Camouflaged Fraudsters", "authors": "Yingtong Dou et al.", "year": 2020, "venue": "CIKM 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2008.08692", "role": "state of the art" }, { "title": "Deep Learning for Anomaly Detection: A Survey", "authors": "Raghavendra Chalapathy et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1901.03407", "role": "survey" } ], "approved": true }, { "id": "sensor-anomaly-detection-and-predictive-maintenance", "name": "Anomaly detection and predictive maintenance on sensor data", "definition": "Detecting abnormal behaviour in machine and process sensor time series to warn of faults before downtime.", "compute_anchor": "TSPulse: pre-trained models of about 1M parameters that run without a GPU; Anomaly Transformer gives no compute.", "data_anchor": "TSPulse reports +20% on the TSB-AD anomaly detection leaderboard across over 75 datasets; NEC SIAT proof of concept at Kalbe Farma (April to June 2026) met two of three targets.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 8 authors on TSPulse; 4 on Anomaly Transformer.", "time_horizon": "Months.", "shelf_life_signals": [ "Tiny pre-trained detectors (TSPulse, 1M parameters) beating models 10 to 100 times larger", "Indonesian industry proofs of concept facilitated by the Ministry of Industry (NEC SIAT at Kalbe Farma)" ], "exemplars": [ "ibm-granite-tspulse", "anomaly-transformer", "nec-siat" ], "observed_actors": [ "nec", "tsinghua-university", "carnegie-mellon-university", "ibm", "mlcommons", "university-of-notre-dame", "sungkyunkwan-university", "microsoft" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "IBM, Microsoft and NEC build detectors and industrial monitoring products.", "b": "Detectors of about 1M parameters (TSPulse) need little compute; Indonesian lab work is not in the sources.", "c": "Anomaly Transformer came from a 4-author team.", "d": "An AI predictive maintenance proof of concept has already run in Indonesia (NEC SIAT at Kalbe Farma, two of three targets met)." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A and vendors release detectors; Tier C benchmarks them on local plant data; Tier D runs predictive maintenance proofs of value in factories.", "sources": [ "https://arxiv.org/abs/2505.13033", "https://arxiv.org/abs/2110.02642", "https://suarapemerintah.id/2026/07/industri-farmasi-masuk-era-ai-kemenperin-fasilitasi-uji-coba-solusi-industri-4-0/" ], "unverified": [], "papers": [ { "title": "LSTM-based Encoder-Decoder for Multi-sensor Anomaly Detection", "authors": "Pankaj Malhotra et al.", "year": 2016, "venue": "ICML 2016 (per arXiv comment)", "url": "https://arxiv.org/abs/1607.00148", "role": "founding" }, { "title": "TranAD: Deep Transformer Networks for Anomaly Detection in Multivariate Time Series Data", "authors": "Shreshth Tuli et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2201.07284", "role": "state of the art" }, { "title": "Anomaly Transformer: Time Series Anomaly Detection with Association Discrepancy", "authors": "Jiehui Xu et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2110.02642", "role": "state of the art" }, { "title": "Current Time Series Anomaly Detection Benchmarks are Flawed and are Creating the Illusion of Progress", "authors": "Renjie Wu et al.", "year": 2020, "venue": "38th IEEE International Conference on Data Engineering (ICDE), 2022, pp. 1479-14", "url": "https://arxiv.org/abs/2009.13807", "role": "state of the art" }, { "title": "Deep Learning for Time Series Anomaly Detection: A Survey", "authors": "Zahra Zamanzadeh Darban et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2211.05244", "role": "survey" } ], "approved": true }, { "id": "deep-learning-architectures-for-time-series", "name": "Deep learning architectures for time-series forecasting and classification", "definition": "Designing neural architectures, from transformers to simple linear and MLP models, for forecasting and classifying time series, measured against statistical baselines.", "compute_anchor": "No compute figures; PatchTST cuts attention compute and memory quadratically by patching; LTSF-Linear is a one-layer linear model.", "data_anchor": "Nine real-life datasets on which one-layer LTSF-Linear beats transformer forecasters (DLinear paper); PatchTST adds masked self-supervised pretraining.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on PatchTST; 4 on the DLinear paper.", "time_horizon": "Months to a year.", "shelf_life_signals": [ "Simple linear baselines beating transformers (DLinear) and pretrained foundation forecasters taking over", "Classical baselines (ETS, Theta, seasonal naive) kept as the yardstick" ], "exemplars": [ "patchtst", "dlinear", "n-beats", "temporal-fusion-transformer" ], "observed_actors": [ "ibm", "tsinghua-university", "google", "intel", "amazon", "carnegie-mellon-university", "element-ai", "mila" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google, Amazon and universities publish forecasting architectures.", "b": "Models of this kind train on single GPUs; the anchor papers come from 4-author teams.", "c": "PatchTST and DLinear each came from 4-author teams.", "d": "Demand and energy forecasting proofs of value use these models with statistical baselines." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A and universities publish architectures; Tier C benchmarks them on Indonesian series; Tier D builds forecasting proofs of value.", "sources": [ "https://arxiv.org/abs/2211.14730", "https://arxiv.org/abs/2205.13504" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "DeepAR: Probabilistic Forecasting with Autoregressive Recurrent Networks", "authors": "David Salinas et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1704.04110", "role": "founding" }, { "title": "iTransformer: Inverted Transformers Are Effective for Time Series Forecasting", "authors": "Yong Liu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2310.06625", "role": "state of the art" }, { "title": "Are Transformers Effective for Time Series Forecasting?", "authors": "Ailing Zeng et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2205.13504", "role": "state of the art" }, { "title": "A Time Series is Worth 64 Words: Long-term Forecasting with Transformers", "authors": "Yuqi Nie et al.", "year": 2022, "venue": "ICLR 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2211.14730", "role": "state of the art" }, { "title": "Transformers in Time Series: A Survey", "authors": "Qingsong Wen et al.", "year": 2022, "venue": "In the 32nd International Joint Conference on Artificial Intelligence (IJCAI 202", "url": "https://arxiv.org/abs/2202.07125", "role": "survey" } ], "approved": true }, { "id": "tinyml-on-microcontrollers", "name": "TinyML on microcontrollers", "definition": "Designing and compressing neural networks and inference engines so that vision, audio and sensor models run on microcontrollers with kilobytes of memory.", "compute_anchor": "MCUNet: over 70% ImageNet top-1 on an off-the-shelf microcontroller, with 3.5 times less SRAM and 5.7 times less Flash than quantised MobileNetV2 and ResNet-18.", "data_anchor": "MLPerf Tiny: four benchmarks (keyword spotting, visual wake words, image classification, anomaly detection), built by more than 50 organisations.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 6 authors on MCUNet; 22 on MLPerf Tiny.", "time_horizon": "One to two years.", "shelf_life_signals": [ "Microcontrollers with neural processing units (STM32N6) widening model sizes", "MLPerf Tiny as the shared yardstick for accuracy, latency and energy" ], "exemplars": [ "mit-han-lab-mcunet", "mlcommons-mlperf-tiny-mobilenet-vww", "stmicroelectronics-st-yolo", "edge-impulse-fomo" ], "observed_actors": [ "stmicroelectronics", "mlcommons", "arm", "silicon-labs", "mit-han-lab", "edge-impulse", "google", "qualcomm" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "The compute is tiny; observed actors are chip vendors and university labs (MIT, Arm, STMicroelectronics) rather than frontier labs.", "b": "Neural architecture search for microcontrollers (MCUNet) needs modest compute; Indonesian work is not in the sources.", "c": "MCUNet came from a 6-author university team.", "d": "Vendor model zoos and MLPerf Tiny reference models deploy on low-cost boards for field proofs of concept." }, "tier_confidence": { "a": "medium", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A and chip vendors supply engines and model zoos; Tier C designs models for local sensors; Tier D deploys them on devices in agriculture, factory and building proofs of value.", "sources": [ "https://arxiv.org/abs/2007.10319", "https://arxiv.org/abs/2106.07597" ], "unverified": [ "compute_anchor (training compute)" ], "papers": [ { "title": "Hello Edge: Keyword Spotting on Microcontrollers", "authors": "Yundong Zhang et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1711.07128", "role": "founding" }, { "title": "MCUNetV2: Memory-Efficient Patch-based Inference for Tiny Deep Learning", "authors": "Ji Lin et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2110.15352", "role": "state of the art" }, { "title": "MLPerf Tiny Benchmark", "authors": "Colby Banbury et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2106.07597", "role": "state of the art" }, { "title": "MCUNet: Tiny Deep Learning on IoT Devices", "authors": "Ji Lin et al.", "year": 2020, "venue": "NeurIPS 2020 (per arXiv comment)", "url": "https://arxiv.org/abs/2007.10319", "role": "state of the art" }, { "title": "TensorFlow Lite Micro: Embedded Machine Learning on TinyML Systems", "authors": "Robert David et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2010.08678", "role": "state of the art" } ], "approved": true }, { "id": "keyword-spotting-and-audio-on-microcontrollers", "name": "Keyword spotting and audio event detection on microcontrollers", "definition": "Always-on speech command, wake-word, noise suppression and sound event models small enough for microcontrollers and DSPs.", "compute_anchor": "Hello Edge: architectures fitted to microcontroller memory and compute; DS-CNN reaches 95.4% accuracy, about 10% above a DNN with a similar parameter count.", "data_anchor": "Speech command datasets for keyword spotting (Hello Edge, NXP and Silicon Labs model zoos); the dataset size is not in the abstract.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on Hello Edge.", "time_horizon": "One to two years.", "shelf_life_signals": [ "Keywords in local languages, which vendor zoos do not cover (vendor examples use English words)", "Audio event detection and noise suppression joining wake words on the same chips" ], "exemplars": [ "arm-hello-edge-kws", "ds-cnn", "mlcommons-mlperf-tiny-ds-cnn-kws", "google-tflm-micro-speech" ], "observed_actors": [ "arm", "silicon-labs", "mlcommons", "stmicroelectronics", "google", "qualcomm", "nxp" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Arm, Google and Qualcomm publish the reference keyword spotters (Hello Edge, TFLite Micro).", "b": "Training a DS-CNN keyword spotter is small; Indonesian keyword data is not in the sources.", "c": "Hello Edge came from a 4-author team.", "d": "Vendor keyword spotters run on low-cost boards for voice-control proofs of concept." }, "tier_confidence": { "a": "medium", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A and vendors publish reference models; Tier C collects Indonesian keyword audio; Tier D builds voice-controlled devices in proofs of value.", "sources": [ "https://arxiv.org/abs/1711.07128" ], "unverified": [ "data_anchor (dataset size)" ], "papers": [ { "title": "Hello Edge: Keyword Spotting on Microcontrollers", "authors": "Yundong Zhang et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1711.07128", "role": "founding" }, { "title": "Speech Commands: A Dataset for Limited-Vocabulary Speech Recognition", "authors": "Pete Warden", "year": 2018, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1804.03209", "role": "founding" }, { "title": "Keyword Transformer: A Self-Attention Model for Keyword Spotting", "authors": "Axel Berg et al.", "year": 2021, "venue": "Proc. Interspeech 2021, 4249-4253", "url": "https://arxiv.org/abs/2104.00769", "role": "state of the art" }, { "title": "Broadcasted Residual Learning for Efficient Keyword Spotting", "authors": "Byeonggeun Kim et al.", "year": 2021, "venue": "INTERSPEECH 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2106.04140", "role": "state of the art" }, { "title": "MatchboxNet: 1D Time-Channel Separable Convolutional Neural Network Architecture for Speech Commands Recognition", "authors": "Somshubra Majumdar et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2004.08531", "role": "state of the art" } ], "approved": true }, { "id": "code-language-models", "name": "Code language models", "definition": "Pretraining and fine-tuning language models on source code for completion, infilling, generation and repair.", "compute_anchor": "StarCoder2: 3B, 7B and 15B models on 3.3 to 4.3 trillion tokens. Code Llama: 7B, 13B, 34B and 70B, trained on 16k-token sequences.", "data_anchor": "The Stack v2 from the Software Heritage archive, 619 programming languages, 4 times larger than the first StarCoder dataset.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 66 authors on StarCoder2; 26 on Code Llama.", "time_horizon": "Months.", "shelf_life_signals": [ "General LLMs absorbing code ability (IBM folds coding into Granite 4.2)", "Open code models from 7B to 30B as completion engines in IDEs" ], "exemplars": [ "bigcode-starcoder2-15b", "meta-code-llama", "alibaba-qwen2-5-coder", "deepseek-deepseek-coder-v2" ], "observed_actors": [ "alibaba", "mistral-ai", "openai", "meta", "cognition", "google", "salesforce", "ibm" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Meta, Alibaba, DeepSeek and Mistral pretrain code models on trillions of tokens (StarCoder2: up to 4.3 trillion).", "b": "Pretraining on 3 to 4 trillion tokens (StarCoder2) is a stretch; fine-tuning an open code model is not; Indonesian lab capacity is not in the sources.", "c": "StarCoder2 needed a 66-author open collaboration, a stretch for one institution unit.", "d": "Open 7B to 15B code models (StarCoder2, Code Llama) serve completion and code assistance proofs of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A and open collaborations pretrain code models; Tier C fine-tunes and evaluates them; Tier D deploys them as assistants in client proofs of value.", "sources": [ "https://arxiv.org/abs/2402.19173", "https://arxiv.org/abs/2308.12950" ], "unverified": [ "compute_anchor (FLOP figure)" ], "papers": [ { "title": "Evaluating Large Language Models Trained on Code", "authors": "Mark Chen et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2107.03374", "role": "founding" }, { "title": "DeepSeek-Coder: When the Large Language Model Meets Programming -- The Rise of Code Intelligence", "authors": "Daya Guo et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2401.14196", "role": "state of the art" }, { "title": "StarCoder 2 and The Stack v2: The Next Generation", "authors": "Anton Lozhkov et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2402.19173", "role": "state of the art" }, { "title": "Qwen2.5-Coder Technical Report", "authors": "Binyuan Hui et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2409.12186", "role": "state of the art" }, { "title": "A Survey on Large Language Models for Code Generation", "authors": "Juyong Jiang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2406.00515", "role": "survey" } ], "approved": true }, { "id": "agentic-software-engineering", "name": "Agentic software engineering", "definition": "Training and evaluating models that act as agents in real repositories, reading code, running tests and submitting fixes for issues.", "compute_anchor": "SWE-agent-LM-32B is fine-tuned from a 32B open model on SWE-smith trajectories; training compute is not in the abstract.", "data_anchor": "SWE-smith: 50k task instances from 128 GitHub repositories, an order of magnitude larger than earlier work; SWE-agent-LM-32B reaches 40.2% Pass@1.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 10 authors on SWE-smith.", "time_horizon": "Months.", "shelf_life_signals": [ "Tool vendors post-training their own coding agents on open bases (Cursor Composer, Cognition SWE records in AURION)", "SWE-bench style leaderboards changing within months" ], "exemplars": [ "swe-bench-team-swe-agent-lm-32b", "all-hands-ai-openhands-lm-32b-v0-1", "mistral-ai-devstral-small-2-24b-instruct-2512", "openai-gpt-5-1-codex" ], "observed_actors": [ "alibaba", "openai", "mistral-ai", "cognition", "anysphere", "all-hands-ai", "anthropic", "google-deepmind" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "OpenAI, Anthropic, Cognition and Cursor post-train coding agents at scale.", "b": "Fine-tuning a 32B model on 50k trajectories (SWE-smith) is a compute stretch; Indonesian capacity is not in the sources.", "c": "SWE-smith came from a 10-author team, but it builds execution environments for 128 repositories.", "d": "Delivery teams can use released coding agents (Devstral, OpenHands LM) on client repositories in proofs of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A trains agents and benchmarks; Tier C builds Indonesian repository tasks; Tier D uses agents to speed client software delivery.", "sources": [ "https://arxiv.org/abs/2504.21798" ], "unverified": [ "compute_anchor (training compute)" ], "papers": [ { "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "authors": "Carlos E. Jimenez et al.", "year": 2023, "venue": "ICLR 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2310.06770", "role": "founding" }, { "title": "SWE-smith: Scaling Data for Software Engineering Agents", "authors": "John Yang et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2504.21798", "role": "state of the art" }, { "title": "AutoCodeRover: Autonomous Program Improvement", "authors": "Yuntong Zhang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2404.05427", "role": "state of the art" }, { "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering", "authors": "John Yang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2405.15793", "role": "state of the art" }, { "title": "Large Language Model-Based Agents for Software Engineering: A Survey", "authors": "Junwei Liu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2409.02977", "role": "survey" } ], "approved": true }, { "id": "hallucination-and-grounding-detection", "name": "Hallucination and grounding detection", "definition": "Small classifiers that check whether a generated claim is supported by its source documents, for retrieval-augmented generation and summarisation.", "compute_anchor": "MiniCheck: small fact-checkers with GPT-4-level performance at 400 times lower cost; parameter count is not in the abstract.", "data_anchor": "Synthetic training data generated with GPT-4 by building realistic factual errors; evaluation on the unified LLM-AggreFact benchmark.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 3 authors on MiniCheck.", "time_horizon": "Months.", "shelf_life_signals": [ "Grounding checks folded into guard models (IBM Granite Guardian RAG groundedness)", "LLM-AggreFact as the shared leaderboard" ], "exemplars": [ "bespoke-labs-bespoke-minicheck-7b", "vectara-hallucination-evaluation-model" ], "observed_actors": [ "ibm", "vectara", "bespoke-labs" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Labs and vendors (Bespoke Labs, Vectara, IBM) release grounding checkers.", "b": "MiniCheck trains on synthetic data from a stronger model, a recipe that does not need frontier compute.", "c": "MiniCheck came from a 3-author team.", "d": "A grounding checker at 400 times lower cost than GPT-4 (MiniCheck) is a practical guard in RAG proofs of value." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A and vendors release checkers and benchmarks; Tier C builds Indonesian grounding test sets; Tier D adds checkers to RAG proofs of value.", "sources": [ "https://arxiv.org/abs/2404.10774" ], "unverified": [ "compute_anchor (parameters)" ], "papers": [ { "title": "TruthfulQA: Measuring How Models Mimic Human Falsehoods", "authors": "Stephanie Lin et al.", "year": 2021, "venue": "ACL 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2109.07958", "role": "founding" }, { "title": "Why Language Models Hallucinate", "authors": "Adam Tauman Kalai et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2509.04664", "role": "state of the art" }, { "title": "SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models", "authors": "Potsawee Manakul et al.", "year": 2023, "venue": "EMNLP 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2303.08896", "role": "state of the art" }, { "title": "Survey of Hallucination in Natural Language Generation", "authors": "Ziwei Ji et al.", "year": 2022, "venue": "ACM Computing Surveys (2022)", "url": "https://arxiv.org/abs/2202.03629", "role": "survey" } ], "approved": true }, { "id": "visual-instruction-tuning", "name": "Visual instruction tuning", "definition": "Connecting a vision encoder to a language model and training it on image-grounded instructions to make an open vision-language assistant.", "compute_anchor": "LLaVA connects a vision encoder and an LLM trained end to end; compute is not given in the abstract.", "data_anchor": "Multimodal instruction data generated with language-only GPT-4; LLaVA scores 85.1% relative to GPT-4 on a synthetic multimodal instruction set and 92.53% on Science QA with GPT-4.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 4 authors on LLaVA.", "time_horizon": "Months.", "shelf_life_signals": [ "Open VLM families updated several times a year (LLaVA, InternVL, Molmo, Qwen-VL)", "Small VLMs under 2B for phones (SmolVLM, FastVLM)" ], "exemplars": [ "llava-llava", "shanghai-ai-lab-internvl", "allen-ai-molmo", "hugging-face-idefics" ], "observed_actors": [ "microsoft", "shanghai-ai-lab", "hugging-face", "google", "salesforce", "zhipu-zai", "alibaba", "google-deepmind" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google, Alibaba, Shanghai AI Lab and Ai2 train open VLMs.", "b": "LLaVA's recipe of an open encoder, an open LLM and generated instruction data needs modest compute; Indonesian VLM work is not in the sources.", "c": "LLaVA came from a 4-author team.", "d": "Open VLMs answer questions about client images in inspection and document proofs of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "high", "d": "high" }, "pipeline_role": "Tier A and open labs release VLMs; Tier B and Tier C tune them on Indonesian images and text; Tier D applies them to inspection and visual question answering.", "sources": [ "https://arxiv.org/abs/2304.08485" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Flamingo: a Visual Language Model for Few-Shot Learning", "authors": "Jean-Baptiste Alayrac et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2204.14198", "role": "founding" }, { "title": "Visual Instruction Tuning", "authors": "Haotian Liu et al.", "year": 2023, "venue": "NeurIPS 2023 (per arXiv comment)", "url": "https://arxiv.org/abs/2304.08485", "role": "founding" }, { "title": "Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution", "authors": "Peng Wang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2409.12191", "role": "state of the art" }, { "title": "Cambrian-1: A Fully Open, Vision-Centric Exploration of Multimodal LLMs", "authors": "Shengbang Tong et al.", "year": 2024, "venue": "NeurIPS 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2406.16860", "role": "state of the art" }, { "title": "A Survey on Multimodal Large Language Models", "authors": "Shukang Yin et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.13549", "role": "survey" } ], "approved": true }, { "id": "document-ocr-and-chart-understanding", "name": "Document, OCR and chart understanding", "definition": "Vision-language models that read text-rich images such as scanned documents, forms, charts and screenshots without a separate OCR pipeline.", "compute_anchor": "CogAgent: 18B parameters with 1120 by 1120 input to read tiny text; MiniCPM-V targets end-side devices such as phones.", "data_anchor": "CogAgent reports state of the art on text-rich benchmarks including TextVQA, ChartQA, InfoVQA and DocVQA; dataset sizes are not in the abstracts.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 23 authors on MiniCPM-V; 14 on CogAgent.", "time_horizon": "Months.", "shelf_life_signals": [ "Small on-device document VLMs (MiniCPM-V on phones)", "OCR-free reading replacing OCR plus layout pipelines in document RAG" ], "exemplars": [ "openbmb-minicpm-v", "microsoft-florence-2", "mistral-ai-pixtral-large", "nvidia-nemotron-nano-12b-v2-vl" ], "observed_actors": [ "google", "openbmb", "microsoft", "zhipu-zai", "mistral-ai", "hugging-face", "adept", "shanghai-ai-lab" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google, Microsoft, Mistral and OpenBMB release document-capable VLMs.", "b": "Fine-tuning an open document VLM on Indonesian forms is within reach; Indonesian lab capacity is not in the sources.", "c": "Building Indonesian document test sets is institution-scale; pretraining an 18B model (CogAgent) is not needed.", "d": "Reading invoices, forms and charts with an open phone-sized VLM (MiniCPM-V) is a common proof of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "high" }, "pipeline_role": "Tier A and open labs release document VLMs; Tier C builds Indonesian document benchmarks; Tier D applies them to invoices, forms and reports.", "sources": [ "https://arxiv.org/abs/2408.01800", "https://arxiv.org/abs/2312.08914" ], "unverified": [ "data_anchor (dataset sizes)" ], "papers": [ { "title": "DocVQA: A Dataset for VQA on Document Images", "authors": "Minesh Mathew et al.", "year": 2020, "venue": "WACV 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2007.00398", "role": "founding" }, { "title": "Qwen2.5-VL Technical Report", "authors": "Shuai Bai et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2502.13923", "role": "state of the art" }, { "title": "DeepSeek-OCR: Contexts Optical Compression", "authors": "Haoran Wei et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2510.18234", "role": "state of the art" }, { "title": "mPLUG-DocOwl 1.5: Unified Structure Learning for OCR-free Document Understanding", "authors": "Anwen Hu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.12895", "role": "state of the art" }, { "title": "OCRBench: On the Hidden Mystery of OCR in Large Multimodal Models", "authors": "Yuliang Liu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.07895", "role": "state of the art" } ], "approved": true }, { "id": "omni-modal-and-unified-multimodal-models", "name": "Omni-modal and unified multimodal models", "definition": "Single models that take text, images, audio and video and produce text and speech, or that both understand and generate images in one transformer.", "compute_anchor": "Qwen2.5-Omni: Thinker-Talker design that streams text and speech; Janus: one autoregressive transformer with decoupled visual encoders; compute is not in the abstracts.", "data_anchor": "Qwen2.5-Omni interleaves audio and video with time-aligned position embeddings (TMRoPE); dataset sizes are not in the abstracts.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 14 authors on Qwen2.5-Omni; 11 on Janus.", "time_horizon": "Months.", "shelf_life_signals": [ "Omni releases from Alibaba, Xiaomi, OpenBMB and NVIDIA within the same year", "Unified understanding and generation (Janus, BAGEL)" ], "exemplars": [ "alibaba-qwen2-5-omni-7b", "deepseek-janus", "openbmb-minicpm-o", "bytedance-bagel" ], "observed_actors": [ "deepseek", "alibaba", "xiaomi", "openbmb", "amazon", "bytedance", "naver", "nvidia" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Alibaba, DeepSeek, ByteDance and Xiaomi train omni and unified models.", "b": "Training across text, image, audio and video at once is a data and compute stretch; Indonesian work is not in the sources.", "c": "Studying released 7B omni models is institution-scale; training one is a stretch.", "d": "Released 7B omni models (Qwen2.5-Omni) support voice and vision assistant proofs of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A trains omni models; Tier C evaluates them in Indonesian; Tier D builds multimodal assistants on released checkpoints.", "sources": [ "https://arxiv.org/abs/2503.20215", "https://arxiv.org/abs/2410.13848" ], "unverified": [ "compute_anchor", "data_anchor (dataset sizes)" ], "papers": [ { "title": "NExT-GPT: Any-to-Any Multimodal LLM", "authors": "Shengqiong Wu et al.", "year": 2023, "venue": "ICML 2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2309.05519", "role": "founding" }, { "title": "Qwen2.5-Omni Technical Report", "authors": "Jin Xu et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2503.20215", "role": "state of the art" }, { "title": "GPT-4o System Card", "authors": "OpenAI et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2410.21276", "role": "state of the art" }, { "title": "Unified-IO 2: Scaling Autoregressive Multimodal Models with Vision, Language, Audio, and Action", "authors": "Jiasen Lu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.17172", "role": "state of the art" }, { "title": "MM-LLMs: Recent Advances in MultiModal Large Language Models", "authors": "Duzhen Zhang et al.", "year": 2024, "venue": "ACL2024 (per arXiv comment)", "url": "https://arxiv.org/abs/2401.13601", "role": "survey" } ], "approved": true }, { "id": "video-understanding-with-language-models", "name": "Video understanding with language models", "definition": "Vision-language models that summarise, search and answer questions about long videos.", "compute_anchor": "VideoLLaMA3 is trained in four stages from vision encoder adaptation to video fine-tuning; compute is not in the abstract.", "data_anchor": "VideoLLaMA3 favours large high-quality image-text data (scenes, documents, charts) over massive video-text data.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 15 authors on VideoLLaMA3.", "time_horizon": "Months.", "shelf_life_signals": [ "Token compression for hour-long video (LongVU)", "Video search products built on video embeddings and video LLMs (TwelveLabs Marengo and Pegasus)" ], "exemplars": [ "alibaba-damo-videollama3-7b", "meta-longvu", "twelvelabs-pegasus" ], "observed_actors": [ "alibaba-damo", "kuaishou", "allen-ai", "hugging-face", "twelvelabs", "google", "google-deepmind", "tsinghua-university" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Alibaba, Meta and Shanghai AI Lab release video LLMs.", "b": "Video training data and compute are a stretch; Indonesian video LLM work is not in the sources.", "c": "Fine-tuning a released 7B video LLM (VideoLLaMA3) on local footage is institution-scale.", "d": "CCTV review, broadcast archive search and site monitoring proofs of concept can use released 7B models." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A releases video LLMs; Tier C adapts them to local video; Tier D applies them to monitoring and archive search.", "sources": [ "https://arxiv.org/abs/2501.13106" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "VideoChat: Chat-Centric Video Understanding", "authors": "KunChang Li et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.06355", "role": "founding" }, { "title": "VideoLLaMA 3: Frontier Multimodal Foundation Models for Image and Video Understanding", "authors": "Boqiang Zhang et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2501.13106", "role": "state of the art" }, { "title": "Video-MME: The First-Ever Comprehensive Evaluation Benchmark of Multi-modal LLMs in Video Analysis", "authors": "Chaoyou Fu et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2405.21075", "role": "state of the art" }, { "title": "LLaVA-Video: Video Instruction Tuning With Synthetic Data", "authors": "Yuanhan Zhang et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2410.02713", "role": "state of the art" }, { "title": "Video Understanding with Large Language Models: A Survey", "authors": "Yolo Y. Tang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2312.17432", "role": "survey" } ], "approved": true }, { "id": "gui-and-computer-use-agents", "name": "GUI and computer-use agents", "definition": "Vision-language models that read screenshots and act on computer or phone interfaces by clicking and typing.", "compute_anchor": "CogAgent: 18B-parameter VLM with 1120 by 1120 input.", "data_anchor": "CogAgent uses only screenshots and outperforms LLM methods that consume extracted HTML text on PC and Android navigation benchmarks (dataset sizes not in the abstract).", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 14 authors on CogAgent.", "time_horizon": "Months.", "shelf_life_signals": [ "Computer-use offerings from OpenAI, Anthropic and Google", "Small open computer-use models (Microsoft Fara-7B)" ], "exemplars": [ "zhipu-zai-cogagent-chat-hf", "microsoft-fara-7b", "openai-computer-use-preview" ], "observed_actors": [ "zhipu-zai", "microsoft", "anthropic", "google-deepmind", "tsinghua-university", "openai" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "OpenAI, Anthropic, Google and Microsoft ship computer-use models.", "b": "Training an 18B screenshot model (CogAgent) with action data is a stretch; Indonesian work is not in the sources.", "c": "Evaluating open computer-use models on Indonesian apps is institution-scale; training is a stretch.", "d": "Released computer-use models automate back-office screens in robotic process automation proofs of concept." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A trains computer-use models; Tier C tests them on Indonesian apps and government portals; Tier D automates client workflows.", "sources": [ "https://arxiv.org/abs/2312.08914" ], "unverified": [ "data_anchor (dataset sizes)" ], "papers": [ { "title": "WebGPT: Browser-assisted question-answering with human feedback", "authors": "Reiichiro Nakano et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2112.09332", "role": "founding" }, { "title": "UI-TARS: Pioneering Automated GUI Interaction with Native Agents", "authors": "Yujia Qin et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2501.12326", "role": "state of the art" }, { "title": "GPT-4V(ision) is a Generalist Web Agent, if Grounded", "authors": "Boyuan Zheng et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2401.01614", "role": "state of the art" }, { "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "authors": "Tianbao Xie et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2404.07972", "role": "state of the art" }, { "title": "GUI Agents: A Survey", "authors": "Dang Nguyen et al.", "year": 2024, "venue": "ACL 2025 (per arXiv comment)", "url": "https://arxiv.org/abs/2412.13501", "role": "survey" } ], "approved": true }, { "id": "medical-vision-language-models", "name": "Medical vision-language models", "definition": "Multimodal models that read medical images and text to answer clinical questions and draft grounded radiology or pathology reports.", "compute_anchor": "MedGemma: medical vision-language models built on Gemma 3 4B and 27B; compute is not in the abstract.", "data_anchor": "MedGemma: 2.6 to 10% gain on medical multimodal question answering and 15.5 to 18.1% on chest X-ray finding classification; MAIRA-2 adds finding localisation and the RadFact evaluation.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 81 authors on MedGemma; 21 on MAIRA-2.", "time_horizon": "One to two years.", "shelf_life_signals": [ "Open medical models on general bases (MedGemma 4B) replacing closed ones", "Grounded report generation with per-sentence checking (MAIRA-2, RadFact)" ], "exemplars": [ "google-medgemma-4b", "microsoft-maira-2", "microsoft-llava-med-v1-5-mistral-7b" ], "observed_actors": [ "google", "microsoft", "google-deepmind", "stanford-university", "paige" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "feasible", "tier_d": "stretch", "tier_rationale": { "a": "Google and Microsoft build medical VLMs with large clinical teams (MedGemma: 81 authors).", "b": "Clinical image data and validation are a stretch for a lab; Indonesian medical VLM work is not in the sources.", "c": "Hospital-linked institution units can fine-tune a released 4B MedGemma on local images.", "d": "Proofs of concept are possible on MedGemma 4B, but clinical validation of the kind MAIRA-2 reports makes value proofs a stretch." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A releases medical VLMs; Tier C validates them with Indonesian hospitals; Tier D builds supervised drafting aids with clinical partners.", "sources": [ "https://arxiv.org/abs/2507.05201", "https://arxiv.org/abs/2406.04449" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Contrastive Learning of Medical Visual Representations from Paired Images and Text", "authors": "Yuhao Zhang et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2010.00747", "role": "founding" }, { "title": "BiomedCLIP: a multimodal biomedical foundation model pretrained from fifteen million scientific image-text pairs", "authors": "Sheng Zhang et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2303.00915", "role": "state of the art" }, { "title": "LLaVA-Med: Training a Large Language-and-Vision Assistant for Biomedicine in One Day", "authors": "Chunyuan Li et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2306.00890", "role": "state of the art" }, { "title": "Towards Generalist Biomedical AI", "authors": "Tao Tu et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2307.14334", "role": "state of the art" }, { "title": "Med-Flamingo: a Multimodal Medical Few-shot Learner", "authors": "Michael Moor et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2307.15189", "role": "state of the art" } ], "approved": true }, { "id": "text-to-text-pretraining-and-summarisation", "name": "Text-to-text pretraining and summarisation", "definition": "Encoder-decoder models pretrained to map any text task to text, with objectives tailored to abstractive summarisation.", "compute_anchor": "T5 compares pretraining objectives, architectures and data sets systematically and combines the findings with scale (the largest release is T5 11B, per its AURION record); compute is not in the abstract.", "data_anchor": "T5: Colossal Clean Crawled Corpus (C4). PEGASUS: gap-sentence pretraining evaluated on 12 summarisation datasets across news, science, emails, patents and bills.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 9 authors on T5; 4 on PEGASUS.", "time_horizon": "Years (mature direction).", "shelf_life_signals": [ "Decoder-only LLMs taking over summarisation", "Encoder-decoder revivals adapted from decoder models (T5Gemma)" ], "exemplars": [ "google-t5", "google-pegasus", "meta-bart", "google-flan-t5" ], "observed_actors": [ "google", "meta", "indonlp", "allen-ai", "carnegie-mellon-university", "hugging-face", "amazon", "cohere-labs" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Google and Meta built T5, PEGASUS and BART.", "b": "Fine-tuning T5-size models for Indonesian summarisation is already done (an Indonesian T5 summariser is in AURION).", "c": "PEGASUS came from a 4-author team.", "d": "Summarisers of T5 or BART size run cheaply for document summarisation proofs of concept." }, "tier_confidence": { "a": "high", "b": "high", "c": "high", "d": "high" }, "pipeline_role": "Tier A publishes pretraining; Tier B and Tier C fine-tune for Indonesian; Tier D summarises client documents.", "sources": [ "https://arxiv.org/abs/1910.10683", "https://arxiv.org/abs/1912.08777" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Get To The Point: Summarization with Pointer-Generator Networks", "authors": "Abigail See et al.", "year": 2017, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1704.04368", "role": "founding" }, { "title": "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer", "authors": "Colin Raffel et al.", "year": 2019, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/1910.10683", "role": "founding" }, { "title": "Scaling Instruction-Finetuned Language Models", "authors": "Hyung Won Chung et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2210.11416", "role": "state of the art" }, { "title": "UL2: Unifying Language Learning Paradigms", "authors": "Yi Tay et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2205.05131", "role": "state of the art" }, { "title": "Pretrained Language Models for Text Generation: A Survey", "authors": "Junyi Li et al.", "year": 2021, "venue": "IJCAI 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2105.10311", "role": "survey" } ], "approved": true }, { "id": "open-post-training-recipes", "name": "Open post-training recipes", "definition": "Openly released data, code and methods (instruction tuning, preference optimisation, verifiable-reward RL) that turn base models into assistants.", "compute_anchor": "Tulu 3 builds on Llama 3.1 base models; compute is not in the abstract.", "data_anchor": "Tulu 3 releases its data, code and recipes, with SFT, DPO and Reinforcement Learning with Verifiable Rewards (RLVR); it beats the instruct versions of Llama 3.1, Qwen 2.5 and Mistral.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 23 authors on Tulu 3.", "time_horizon": "Months.", "shelf_life_signals": [ "Fully open releases (OLMo, Tulu) with data and code", "RLVR moving from labs to open recipes" ], "exemplars": [ "allen-ai-tulu", "hugging-face-zephyr-7b-beta", "google-flan-t5", "bigscience-bloomz" ], "observed_actors": [ "allen-ai", "google", "bigscience", "indonlp", "hugging-face", "intel", "adept", "cohere-labs" ], "tier_a": "feasible", "tier_b": "feasible", "tier_c": "feasible", "tier_d": "feasible", "tier_rationale": { "a": "Ai2, Hugging Face and Google publish the recipes.", "b": "Open recipes (Tulu 3) let an Indonesian lab post-train an open base without inventing methods.", "c": "Applying a released recipe to an 8B base is institution-scale.", "d": "Delivery teams can apply SFT and DPO recipes for client-specific assistants." }, "tier_confidence": { "a": "high", "b": "high", "c": "medium", "d": "medium" }, "pipeline_role": "Tier A and open labs release recipes; Tier B applies them to Indonesian models; Tier D tunes client assistants.", "sources": [ "https://arxiv.org/abs/2411.15124" ], "unverified": [ "compute_anchor" ], "papers": [ { "title": "Training language models to follow instructions with human feedback", "authors": "Long Ouyang et al.", "year": 2022, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2203.02155", "role": "founding" }, { "title": "Tulu 3: Pushing Frontiers in Open Language Model Post-Training", "authors": "Nathan Lambert et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2411.15124", "role": "state of the art" }, { "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model", "authors": "Rafael Rafailov et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2305.18290", "role": "state of the art" }, { "title": "Zephyr: Direct Distillation of LM Alignment", "authors": "Lewis Tunstall et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2310.16944", "role": "state of the art" }, { "title": "Open Problems and Fundamental Limitations of Reinforcement Learning from Human Feedback", "authors": "Stephen Casper et al.", "year": 2023, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2307.15217", "role": "survey" } ], "approved": true }, { "id": "diffusion-language-models", "name": "Diffusion language models", "definition": "Language models that generate many tokens in parallel by iterative denoising instead of one token at a time.", "compute_anchor": "Mercury Coder Mini and Small reach 1,109 and 737 tokens per second on NVIDIA H100 GPUs. DiffuCoder: 7B trained on 130B tokens of code.", "data_anchor": "DiffuCoder: 130B tokens of code; Mercury: code benchmarks and Copilot Arena ranking.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 13 authors on Mercury; 7 on DiffuCoder.", "time_horizon": "Months to a year.", "shelf_life_signals": [ "Commercial diffusion LLMs (Inception Mercury 2) and open ones (DiffusionGemma, DiffuCoder)", "Throughput claims of up to 10 times speed-optimised autoregressive models (Mercury)" ], "exemplars": [ "inception-labs-mercury", "apple-diffucoder-7b-instruct", "google-diffusiongemma-26b-a4b-it" ], "observed_actors": [ "inception-labs", "apple", "google" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "feasible", "tier_rationale": { "a": "Inception, Google and Apple train diffusion LLMs.", "b": "Training a 7B model on 130B tokens (DiffuCoder) is a stretch; Indonesian work is not in the sources.", "c": "Studying released diffusion LLMs is institution-scale; training one is a stretch.", "d": "Diffusion LLMs are reachable through APIs and open checkpoints where latency matters; delivery evidence is not in the sources." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "low" }, "pipeline_role": "Tier A trains diffusion LLMs; Tier C studies their decoding; Tier D tests them where low latency matters.", "sources": [ "https://arxiv.org/abs/2506.17298", "https://arxiv.org/abs/2506.20639" ], "unverified": [], "papers": [ { "title": "Structured Denoising Diffusion Models in Discrete State-Spaces", "authors": "Jacob Austin et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2107.03006", "role": "founding" }, { "title": "Large Language Diffusion Models", "authors": "Shen Nie et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2502.09992", "role": "state of the art" }, { "title": "Mercury: Ultra-Fast Language Models Based on Diffusion", "authors": "Inception Labs et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2506.17298", "role": "state of the art" }, { "title": "Dream 7B: Diffusion Large Language Models", "authors": "Jiacheng Ye et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2508.15487", "role": "state of the art" }, { "title": "A Survey on Diffusion Language Models", "authors": "Tianyi Li et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2508.10875", "role": "survey" } ], "approved": true }, { "id": "mathematical-reasoning-and-formal-theorem-proving", "name": "Mathematical reasoning and formal theorem proving", "definition": "Training models to solve mathematics problems and to write machine-checked proofs in systems such as Lean 4.", "compute_anchor": "DeepSeekMath 7B: continued pretraining on 120B math tokens with GRPO. DeepSeek-Prover-V2: 671B parameters.", "data_anchor": "120B math tokens selected from Common Crawl (DeepSeekMath, 51.7% on MATH); DeepSeek-Prover-V2 reaches 88.9% on MiniF2F-test and solves 49 of 658 PutnamBench problems.", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 11 authors on DeepSeekMath; 18 on DeepSeek-Prover-V2.", "time_horizon": "Months.", "shelf_life_signals": [ "GRPO (from DeepSeekMath) adopted widely for reasoning RL", "Formal proof models scoring on PutnamBench" ], "exemplars": [ "deepseek-deepseek-math-7b-instruct", "deepseek-deepseek-prover-v2-7b", "absolute-zero" ], "observed_actors": [ "deepseek", "tsinghua-university" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "DeepSeek trains math and prover models up to 671B parameters.", "b": "Continued pretraining on 120B math tokens (DeepSeekMath) is a stretch; Indonesian work is not in the sources.", "c": "Fine-tuning a released 7B prover is institution-scale; the 120B-token pretraining is a stretch.", "d": "The sources show no delivery use case for formal proving." }, "tier_confidence": { "a": "high", "b": "medium", "c": "medium", "d": "low" }, "pipeline_role": "Tier A trains math and prover models; Tier C uses released ones in teaching and research; Tier D draws on the resulting reasoning gains in general models.", "sources": [ "https://arxiv.org/abs/2402.03300", "https://arxiv.org/abs/2504.21801" ], "unverified": [], "papers": [ { "title": "Generative Language Modeling for Automated Theorem Proving", "authors": "Stanislas Polu et al.", "year": 2020, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2009.03393", "role": "founding" }, { "title": "Measuring Mathematical Problem Solving With the MATH Dataset", "authors": "Dan Hendrycks et al.", "year": 2021, "venue": "NeurIPS 2021 (per arXiv comment)", "url": "https://arxiv.org/abs/2103.03874", "role": "founding" }, { "title": "DeepSeek-Prover-V2: Advancing Formal Mathematical Reasoning via Reinforcement Learning for Subgoal Decomposition", "authors": "Z. Z. Ren et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2504.21801", "role": "state of the art" }, { "title": "Seed-Prover: Deep and Broad Reasoning for Automated Theorem Proving", "authors": "Luoxin Chen et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2507.23726", "role": "state of the art" }, { "title": "A Survey on Deep Learning for Theorem Proving", "authors": "Zhaoyu Li et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2404.09939", "role": "survey" } ], "approved": true }, { "id": "differentially-private-language-model-training", "name": "Differentially private language model training", "definition": "Training language models with differential privacy, adding calibrated noise during training so the model cannot memorise any single training sequence.", "compute_anchor": "VaultGemma 1B: 1 billion parameters, pretrained fully with differential privacy; Google used DP scaling laws to set the compute for a compute-optimal 1B model and found DP training needs a much smaller model with a much larger batch size than non-private training. Total training FLOPs are not in the sources.", "data_anchor": "Same training mixture as Gemma 2, split into sequences of 1,024 tokens; sequence-level guarantee of epsilon at most 2.0 and delta at most 1.1e-10. Google reports utility comparable to non-private models from roughly five years ago (a GPT-2 class baseline).", "team_size_proxy": "Proxy only (author count of an exemplar paper, not a headcount): 21 authors on the VaultGemma paper (arXiv 2510.15001).", "time_horizon": "Years.", "shelf_life_signals": [ "Gap between DP and non-private models of the same size narrowing in later releases", "User-level rather than sequence-level DP guarantees appearing in released models" ], "exemplars": [ "google-vaultgemma-1b" ], "observed_actors": [ "google" ], "tier_a": "feasible", "tier_b": "stretch", "tier_c": "stretch", "tier_d": "not feasible", "tier_rationale": { "a": "Google pretrained the 1B VaultGemma fully with differential privacy on the Gemma 2 data mixture.", "b": "DP pretraining needs much larger batch sizes and compute than non-private training of the same model (Google blog); Indonesian work on it is not in the sources.", "c": "Training at 1B scale with DP batch sizes is a stretch for an institution unit; DP fine-tuning costs are not in the sources.", "d": "Pretraining with DP is beyond a delivery team; using the released VaultGemma 1B is the delivery path, and its utility matches roughly five-year-old non-private models." }, "tier_confidence": { "a": "high", "b": "low", "c": "low", "d": "medium" }, "pipeline_role": "Tier A trains and releases DP models and scaling laws; Tier C studies privacy guarantees and audits; Tier D uses released DP models where training data must not be memorised.", "sources": [ "https://arxiv.org/abs/2510.15001", "https://research.google/blog/vaultgemma-the-worlds-most-capable-differentially-private-llm/" ], "unverified": [ "compute_anchor (total training FLOPs)" ], "papers": [ { "title": "Deep Learning with Differential Privacy", "authors": "Martín Abadi et al.", "year": 2016, "venue": "Proceedings of the 2016 ACM SIGSAC Conference on Computer and Communications Sec", "url": "https://arxiv.org/abs/1607.00133", "role": "founding" }, { "title": "VaultGemma: A Differentially Private Gemma Model", "authors": "Amer Sinha et al.", "year": 2025, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2510.15001", "role": "state of the art" }, { "title": "Large Language Models Can Be Strong Differentially Private Learners", "authors": "Xuechen Li et al.", "year": 2021, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2110.05679", "role": "state of the art" }, { "title": "Differentially Private Fine-tuning of Language Models", "authors": "Da Yu et al.", "year": 2021, "venue": "ICLR 2022 (per arXiv comment)", "url": "https://arxiv.org/abs/2110.06500", "role": "state of the art" }, { "title": "On Protecting the Data Privacy of Large Language Models (LLMs): A Survey", "authors": "Biwei Yan et al.", "year": 2024, "venue": "arXiv preprint", "url": "https://arxiv.org/abs/2403.05156", "role": "survey" } ], "approved": true } ]