{"id":"transformer","title":"Transformer architecture","url":"https://tracetheroots.com/transformer-architecture","year":2017,"approximate":false,"when":"2017","date":"2017-06-12","era":"Information Age","eraId":"info","category":"Technology","categoryId":"tec","summary":"Google researchers publish \"Attention Is All You Need,\" the design behind modern AI language models.","turningPoint":true,"causes":[{"id":"deeplearning","title":"Deep learning breakthrough","url":"https://tracetheroots.com/deep-learning-breakthrough"},{"id":"attention","title":"Neural attention for translation","url":"https://tracetheroots.com/neural-attention-for-translation"}],"effects":[{"id":"gpt3","title":"GPT-3","url":"https://tracetheroots.com/gpt-3"},{"id":"diffusion","title":"AI image generators","url":"https://tracetheroots.com/ai-image-generators"},{"id":"chatgpt","title":"ChatGPT and generative AI","url":"https://tracetheroots.com/chatgpt"}],"mainLine":[{"id":"bigbang","title":"The Big Bang","url":"https://tracetheroots.com/big-bang"},{"id":"firststars","title":"First stars ignite","url":"https://tracetheroots.com/first-stars-ignite"},{"id":"supernovae","title":"Supernovae seed heavy elements","url":"https://tracetheroots.com/supernovae-seed-heavy-elements"},{"id":"sun","title":"The Sun forms","url":"https://tracetheroots.com/the-sun-forms"},{"id":"earth","title":"Earth forms","url":"https://tracetheroots.com/earth-forms"},{"id":"oceans","title":"First oceans","url":"https://tracetheroots.com/first-oceans"},{"id":"life","title":"First life","url":"https://tracetheroots.com/first-life"},{"id":"photosyn","title":"Photosynthesis evolves","url":"https://tracetheroots.com/photosynthesis-evolves"},{"id":"chloroplasts","title":"Algae gain chloroplasts","url":"https://tracetheroots.com/algae-gain-chloroplasts"},{"id":"landplants","title":"Plants colonize land","url":"https://tracetheroots.com/plants-colonize-land"},{"id":"forests","title":"First forests","url":"https://tracetheroots.com/first-forests"},{"id":"coalforests","title":"Carboniferous coal forests","url":"https://tracetheroots.com/carboniferous-coal-forests"},{"id":"coalmining","title":"Britain turns to coal","url":"https://tracetheroots.com/britain-turns-to-coal"},{"id":"newcomen","title":"Newcomen steam engine","url":"https://tracetheroots.com/newcomen-steam-engine"},{"id":"watt","title":"Watt's steam engine","url":"https://tracetheroots.com/watts-steam-engine"},{"id":"thermo","title":"Thermodynamics","url":"https://tracetheroots.com/thermodynamics"},{"id":"entropy","title":"Entropy","url":"https://tracetheroots.com/entropy"},{"id":"planck","title":"Quantum hypothesis","url":"https://tracetheroots.com/quantum-hypothesis"},{"id":"qm","title":"Quantum mechanics","url":"https://tracetheroots.com/quantum-mechanics"},{"id":"transistor","title":"Transistor","url":"https://tracetheroots.com/transistor"},{"id":"ic","title":"Integrated circuit","url":"https://tracetheroots.com/integrated-circuit"},{"id":"microprocessor","title":"Microprocessor","url":"https://tracetheroots.com/microprocessor"},{"id":"gpu","title":"GPU","url":"https://tracetheroots.com/gpu"},{"id":"deeplearning","title":"Deep learning breakthrough","url":"https://tracetheroots.com/deep-learning-breakthrough"},{"id":"transformer","title":"Transformer architecture","url":"https://tracetheroots.com/transformer-architecture"}],"sources":[{"title":"Attention Is All You Need","url":"https://arxiv.org/abs/1706.03762"},{"title":"Attention is All You Need","url":"https://research.google/pubs/attention-is-all-you-need/"},{"title":"Transformer co-author Llion Jones leaves Google for startup Sakana AI","url":"https://www.cnbc.com/2023/08/17/transformer-co-author-llion-jones-leaves-google-for-startup-sakana-ai.html"},{"title":"Neural Machine Translation by Jointly Learning to Align and Translate","url":"https://arxiv.org/abs/1409.0473"}]}