{"id":"attention","title":"Neural attention for translation","url":"https://tracetheroots.com/neural-attention-for-translation","year":2014,"approximate":false,"when":"2014","date":"2014-09-01","era":"Information Age","eraId":"info","category":"Technology","categoryId":"tec","summary":"Dzmitry Bahdanau, Kyunghyun Cho and Yoshua Bengio let a translation network learn which input words to focus on. Attention becomes the core of the Transformer.","turningPoint":false,"causes":[{"id":"lstm","title":"Long short-term memory (LSTM)","url":"https://tracetheroots.com/long-short-term-memory-lstm"},{"id":"deeplearning","title":"Deep learning breakthrough","url":"https://tracetheroots.com/deep-learning-breakthrough"}],"effects":[{"id":"transformer","title":"Transformer architecture","url":"https://tracetheroots.com/transformer-architecture"}],"mainLine":[{"id":"bigbang","title":"The Big Bang","url":"https://tracetheroots.com/big-bang"},{"id":"firststars","title":"First stars ignite","url":"https://tracetheroots.com/first-stars-ignite"},{"id":"supernovae","title":"Supernovae seed heavy elements","url":"https://tracetheroots.com/supernovae-seed-heavy-elements"},{"id":"sun","title":"The Sun forms","url":"https://tracetheroots.com/the-sun-forms"},{"id":"earth","title":"Earth forms","url":"https://tracetheroots.com/earth-forms"},{"id":"oceans","title":"First oceans","url":"https://tracetheroots.com/first-oceans"},{"id":"life","title":"First life","url":"https://tracetheroots.com/first-life"},{"id":"photosyn","title":"Photosynthesis evolves","url":"https://tracetheroots.com/photosynthesis-evolves"},{"id":"chloroplasts","title":"Algae gain chloroplasts","url":"https://tracetheroots.com/algae-gain-chloroplasts"},{"id":"landplants","title":"Plants colonize land","url":"https://tracetheroots.com/plants-colonize-land"},{"id":"forests","title":"First forests","url":"https://tracetheroots.com/first-forests"},{"id":"coalforests","title":"Carboniferous coal forests","url":"https://tracetheroots.com/carboniferous-coal-forests"},{"id":"coalmining","title":"Britain turns to coal","url":"https://tracetheroots.com/britain-turns-to-coal"},{"id":"newcomen","title":"Newcomen steam engine","url":"https://tracetheroots.com/newcomen-steam-engine"},{"id":"watt","title":"Watt's steam engine","url":"https://tracetheroots.com/watts-steam-engine"},{"id":"thermo","title":"Thermodynamics","url":"https://tracetheroots.com/thermodynamics"},{"id":"entropy","title":"Entropy","url":"https://tracetheroots.com/entropy"},{"id":"planck","title":"Quantum hypothesis","url":"https://tracetheroots.com/quantum-hypothesis"},{"id":"qm","title":"Quantum mechanics","url":"https://tracetheroots.com/quantum-mechanics"},{"id":"transistor","title":"Transistor","url":"https://tracetheroots.com/transistor"},{"id":"ic","title":"Integrated circuit","url":"https://tracetheroots.com/integrated-circuit"},{"id":"microprocessor","title":"Microprocessor","url":"https://tracetheroots.com/microprocessor"},{"id":"gpu","title":"GPU","url":"https://tracetheroots.com/gpu"},{"id":"deeplearning","title":"Deep learning breakthrough","url":"https://tracetheroots.com/deep-learning-breakthrough"},{"id":"attention","title":"Neural attention for translation","url":"https://tracetheroots.com/neural-attention-for-translation"}],"sources":[{"title":"Neural Machine Translation by Jointly Learning to Align and Translate","url":"https://arxiv.org/pdf/1409.0473"},{"title":"Effective Approaches to Attention-based Neural Machine Translation","url":"https://arxiv.org/pdf/1508.04025v2"},{"title":"Attention Model","url":"https://www2.statmt.org/survey/Topic/AttentionModel"}]}