[{"content":{"TLDR":"We show language understanding via reading is promising way to learn policies that generalise to new environments.","abstract":" ","authors":["Victor Zhong","Tim Rockt\u00e4schel","Edward Grefenstette"],"iclr_id":"SJgob6NKvH","keywords":["nlp","reading comprehension","reasoning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"RTFM: Generalising to New Environment Dynamics via Reading"},"forum":"SJgob6NKvH","id":"SJgob6NKvH"},{"content":{"TLDR":"As the performance of computer systems stagnates due to the end of Moore\u2019s Law,\nthere is a need for new models that can understand and optimize the execution\nof general purpose code. While there is a growing body of work on using Graph\nNeural Network...","abstract":" ","authors":["Zhan Shi","Kevin Swersky","Daniel Tarlow","Parthasarathy Ranganathan","Milad Hashemi"],"iclr_id":"SJetQpEYvB","keywords":["graph embedding","graph networks","memory","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"LEARNING EXECUTION THROUGH NEURAL CODE FUSION"},"forum":"SJetQpEYvB","id":"SJetQpEYvB"},{"content":{"TLDR":"Neural text generation is a key tool in natural language applications, but it is well known there are major problems at its core. In particular, standard likelihood training and decoding leads to dull and repetitive outputs. While some post-hoc fixes...","abstract":" ","authors":["Sean Welleck","Ilia Kulikov","Stephen Roller","Emily Dinan","Kyunghyun Cho","Jason Weston"],"iclr_id":"SJeYe0NtvH","keywords":["generation","language modeling","nlp","text generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Neural Text Generation With Unlikelihood Training"},"forum":"SJeYe0NtvH","id":"SJeYe0NtvH"},{"content":{"TLDR":"We propose SNOW, an efficient way of transfer and lifelong learning by subscribing knowledge of a source model for new tasks through a novel channel pooling block.","abstract":" ","authors":["Chungkuk Yoo","Bumsoo Kang","Minsik Cho"],"iclr_id":"rJxtgJBKDr","keywords":["cnn","lifelong learning","multi task","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"SNOW: Subscribing to Knowledge via Channel Pooling for Transfer & Lifelong Learning of Convolutional Neural Networks"},"forum":"rJxtgJBKDr","id":"rJxtgJBKDr"},{"content":{"TLDR":"We present a learning rule for feedback weights in a spiking neural network that addresses the weight transport problem.","abstract":" ","authors":["Jordan Guerguiev","Konrad Kording","Blake Richards"],"iclr_id":"rJxWxxSYvB","keywords":["feedback alignment","gradient descent","regression"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Spike-based causal inference for weight alignment"},"forum":"rJxWxxSYvB","id":"rJxWxxSYvB"},{"content":{"TLDR":"We introduce an efficient quantization process that allows for performance acceleration on specialized integer-only neural network accelerator.","abstract":" ","authors":["Xiandong Zhao","Ying Wang","Xuyi Cai","Cheng Liu","Lei Zhang"],"iclr_id":"H1lBj2VFPS","keywords":["acceleration","batch normalization","quantization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Linear Symmetric Quantization of Neural Networks for Low-precision Integer Hardware"},"forum":"H1lBj2VFPS","id":"H1lBj2VFPS"},{"content":{"TLDR":"We propose a novel reinforcement learning-based skill transfer and composition method that takes the agent's primitive policies to solve unseen tasks.","abstract":" ","authors":["Ahmed H. Qureshi","Jacob J. Johnson","Yuzhe Qin","Taylor Henderson","Byron Boots","Michael C. Yip"],"iclr_id":"H1ezFREtwH","keywords":["planning","reinforcement learning","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Composing Task-Agnostic Policies with Deep Reinforcement Learning"},"forum":"H1ezFREtwH","id":"H1ezFREtwH"},{"content":{"TLDR":"We introduce MetaGenRL, a novel meta reinforcement learning algorithm. Unlike prior work, MetaGenRL can generalize to new environments that are entirely different from those used for meta-training.","abstract":" ","authors":["Louis Kirsch","Sjoerd van Steenkiste","Juergen Schmidhuber"],"iclr_id":"S1evHerYPr","keywords":["generalization","meta learning","meta reinforcement learning","reinforcement learning","sample efficiency"],"recs":[],"session":["Monday: Meta-learning"],"session_links":[null],"session_times":[null],"title":"Improving Generalization in Meta Reinforcement Learning using Learned Objectives"},"forum":"S1evHerYPr","id":"S1evHerYPr"},{"content":{"TLDR":"We present an image segmentation method that iteratively evolves a polygon. At each iteration, the vertices of the polygon are displaced based on the local value of a 2D shift map that is inferred from the input image via an encoder-decoder architect...","abstract":" ","authors":["Shir Gur","Tal Shaharabany","Lior Wolf"],"iclr_id":"rkxawlHKDr","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"End to End Trainable Active Contours via Differentiable Rendering"},"forum":"rkxawlHKDr","id":"rkxawlHKDr"},{"content":{"TLDR":"We introduce a notion of conservatively-extrapolated value functions, which provably lead to policies that can self-correct to stay close to the demonstration states, and learn them with a novel negative sampling technique.","abstract":" ","authors":["Yuping Luo","Huazhe Xu","Tengyu Ma"],"iclr_id":"rke-f6NKvS","keywords":["imitation learning","model based rl","reinforcement learning","robotics","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning Self-Correctable Policies and Value Functions from Demonstrations with Negative Sampling"},"forum":"rke-f6NKvS","id":"rke-f6NKvS"},{"content":{"TLDR":"We introduce a method to automatically discover task-agnostic options that encourage exploration for reinforcement learning.","abstract":" ","authors":["Yuu Jinnai","Jee Won Park","Marlos C. Machado","George Konidaris"],"iclr_id":"SkeIyaVtwB","keywords":["reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Exploration in Reinforcement Learning with Deep Covering Options"},"forum":"SkeIyaVtwB","id":"SkeIyaVtwB"},{"content":{"TLDR":"We prove that for a large class of functions f there exists an interval certified robust network approximating f up to arbitrary precision.","abstract":" ","authors":["Maximilian Baader","Matthew Mirman","Martin Vechev"],"iclr_id":"B1gX8kBtPr","keywords":["adversarial","adversarial attacks","interval bound propagation","relu networks","robustness","universal approximation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Universal Approximation with Certified Networks"},"forum":"B1gX8kBtPr","id":"B1gX8kBtPr"},{"content":{"TLDR":"Handling missing data is one of the most fundamental problems in machine learning. Among many approaches, the simplest and most intuitive way is zero imputation, which treats the value of a missing entry simply as zero. However, many studies have exp...","abstract":" ","authors":["Joonyoung Yi","Juhyuk Lee","Kwang Joon Kim","Sung Ju Hwang","Eunho Yang"],"iclr_id":"BylsKkHYvH","keywords":["adversarial","collaborative filtering","missing data","tabular data"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Why Not to Use Zero Imputation? Correcting Sparsity Bias in Training Neural Networks"},"forum":"BylsKkHYvH","id":"BylsKkHYvH"},{"content":{"TLDR":"While there are convergence guarantees for temporal difference (TD) learning when using linear function approximators, the situation for nonlinear models is far less understood, and divergent examples are known. Here we take a first step towards exte...","abstract":" ","authors":["David Brandfonbrener","Joan Bruna"],"iclr_id":"SJezGp4YPr","keywords":["reinforcement learning","relu networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Geometric Insights into the Convergence of Nonlinear TD Learning"},"forum":"SJezGp4YPr","id":"SJezGp4YPr"},{"content":{"TLDR":"We proposed a normalization layer for GNN models to solve the oversmoothing problem.","abstract":" ","authors":["Lingxiao Zhao","Leman Akoglu"],"iclr_id":"rkecl1rtwB","keywords":["graph networks","normalization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"PairNorm: Tackling Oversmoothing in GNNs"},"forum":"rkecl1rtwB","id":"rkecl1rtwB"},{"content":{"TLDR":"We study the landscape of squared loss in neural networks with one-hidden layer and ReLU activation functions.  Let $m$ and $d$ be the widths of hidden and input layers, respectively. We show that there exist poor local minima with positive curvature...","abstract":" ","authors":["Arsalan Sharifnassab","Saber Salehkaleybar","S. Jamaloddin Golestani"],"iclr_id":"BkgXHTNtvS","keywords":["deep learning theory","loss landscape","optimization","over parameterization","relu networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Bounds on Over-Parameterization for Guaranteed Existence of Descent Paths in Shallow ReLU Networks"},"forum":"BkgXHTNtvS","id":"BkgXHTNtvS"},{"content":{"TLDR":"We propose a new stream of adversarial training approach called Robust Local Features for Adversarial Training (RLFAT) that significantly improves both the adversarially robust generalization and the standard generalization.","abstract":" ","authors":["Chuanbiao Song","Kun He","Jiadong Lin","Liwei Wang","John E. Hopcroft"],"iclr_id":"H1lZJpVFvr","keywords":["adversarial","adversarial example","generalization","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Robust Local Features for Improving the Generalization of Adversarial Training"},"forum":"H1lZJpVFvr","id":"H1lZJpVFvr"},{"content":{"TLDR":"A novel loss bridges curriculum learning and robust learning","abstract":" ","authors":["Yueming Lyu","Ivor W. Tsang"],"iclr_id":"rkgt0REKwS","keywords":["curriculum learning","expressive power","generalization","robust learning","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Curriculum Loss: Robust Learning and Generalization  against Label Corruption"},"forum":"rkgt0REKwS","id":"rkgt0REKwS"},{"content":{"TLDR":"The recently proposed BERT (Devlin et al., 2019) has shown great power on a variety of natural language understanding tasks, such as text classification, reading comprehension, etc. However, how to effectively apply BERT to neural machine translation...","abstract":" ","authors":["Jinhua Zhu","Yingce Xia","Lijun Wu","Di He","Tao Qin","Wengang Zhou","Houqiang Li","Tieyan Liu"],"iclr_id":"Hyl7ygStwB","keywords":["attention","fine tuning","machine translation","nlp","reading comprehension","text classification","transformer","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Incorporating BERT into Neural Machine Translation"},"forum":"Hyl7ygStwB","id":"Hyl7ygStwB"},{"content":{"TLDR":"A novel Bayesian deep learning framework that captures and relates hierarchical semantic and visual concepts, performing well on a variety of image and text modeling and generation tasks.","abstract":" ","authors":["Hao Zhang","Bo Chen","Long Tian","Zhengjue Wang","Mingyuan Zhou"],"iclr_id":"H1x5wRVtvS","keywords":["adversarial","gan","generation","generative models","image generation","text generation","zero shot learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Variational Hetero-Encoder Randomized GANs for Joint Image-Text Modeling"},"forum":"H1x5wRVtvS","id":"H1x5wRVtvS"},{"content":{"TLDR":"Over-parameterized deep neural networks trained by simple first-order methods are known to be able to fit any labeling of data. Such over-fitting ability hinders generalization when mislabeled training examples are present. On the other hand, simple ...","abstract":" ","authors":["Wei Hu","Zhiyuan Li","Dingli Yu"],"iclr_id":"Hke3gyHYwH","keywords":["deep learning theory","generalization","gradient descent","neural tangent kernel","noisy labels","regularization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Simple and Effective Regularization Methods for Training on Noisily Labeled Data with Generalization Guarantee"},"forum":"Hke3gyHYwH","id":"Hke3gyHYwH"},{"content":{"TLDR":"Parameter-transfer is a well-known and versatile approach for meta-learning, with applications including few-shot learning, federated learning, with personalization, and reinforcement learning. However, parameter-transfer algorithms often require sha...","abstract":" ","authors":["Jeffrey Li","Mikhail Khodak","Sebastian Caldas","Ameet Talwalkar"],"iclr_id":"rJgqMRVYvr","keywords":["federated learning","fewshot learning","language modeling","meta learning","nlp","privacy","reinforcement learning","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Differentially Private Meta-Learning"},"forum":"rJgqMRVYvr","id":"rJgqMRVYvr"},{"content":{"TLDR":"We introduce Deep SAD, a deep method for general semi-supervised anomaly detection that especially takes advantage of labeled anomalies.","abstract":" ","authors":["Lukas Ruff","Robert A. Vandermeulen","Nico G\u00f6rnitz","Alexander Binder","Emmanuel M\u00fcller","Klaus-Robert M\u00fcller","Marius Kloft"],"iclr_id":"HkgH0TEYwH","keywords":["anomaly detection","outlier detection","semi supervised learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep Semi-Supervised Anomaly Detection"},"forum":"HkgH0TEYwH","id":"HkgH0TEYwH"},{"content":{"TLDR":"We show how to automatically learn dynamical distances in reinforcement learning setting and use them to provide well-shaped reward functions for reaching new goals.","abstract":" ","authors":["Kristian Hartikainen","Xinyang Geng","Tuomas Haarnoja","Sergey Levine"],"iclr_id":"H1lmhaVtvr","keywords":["reinforcement learning","robotics","semi supervised learning","skill discovery","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Dynamical Distance Learning for Semi-Supervised and Unsupervised Skill Discovery"},"forum":"H1lmhaVtvr","id":"H1lmhaVtvr"},{"content":{"TLDR":"Generative adversarial networks (GANs) have attracted huge attention due to\nits capability to generate visual realistic images. However, most of the existing\nmodels suffer from the mode collapse or mode mixture problems. In this work, we\ngive a theor...","abstract":" ","authors":["Dongsheng An","Yang Guo","Na Lei","Zhongxuan Luo","Shing-Tung Yau","Xianfeng Gu"],"iclr_id":"HkldyTNYwH","keywords":["adversarial","attention","autoencoder","gan","generative models","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"AE-OT: A NEW GENERATIVE MODEL BASED ON EXTENDED SEMI-DISCRETE OPTIMAL TRANSPORT"},"forum":"HkldyTNYwH","id":"HkldyTNYwH"},{"content":{"TLDR":"We obtain state-of-the-art on robustness to data shifts, and we maintain calibration under data shift even though even when accuracy drops","abstract":" ","authors":["Dan Hendrycks","Norman Mu","Ekin Dogus Cubuk","Barret Zoph","Justin Gilmer","Balaji Lakshminarayanan"],"iclr_id":"S1gmrxHFvB","keywords":["distributed","robustness","uncertainty"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty"},"forum":"S1gmrxHFvB","id":"S1gmrxHFvB"},{"content":{"TLDR":"We present a new method that combines transfer-based and scored black-box adversarial attack, improving the success rate and query efficiency of black-box adversarial attack across different network architectures.","abstract":" ","authors":["Zhichao Huang","Tong Zhang"],"iclr_id":"SJxhNTNYwB","keywords":["adversarial","imagenet","perturbation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Black-Box Adversarial Attack with Transferable Model-based Embedding"},"forum":"SJxhNTNYwB","id":"SJxhNTNYwB"},{"content":{"TLDR":"Perturbations can be used to train feedback weights to learn in fully connected and convolutional neural networks","abstract":" ","authors":["Benjamin James Lansdell","Prashanth Ravi Prakash","Konrad Paul Kording"],"iclr_id":"ByeUBANtvB","keywords":["cnn","feedback alignment","perturbation","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning to solve the credit assignment problem"},"forum":"ByeUBANtvB","id":"ByeUBANtvB"},{"content":{"TLDR":"We introduced a pruning criterion for pruning networks before training by preserving gradient flow.","abstract":" ","authors":["Chaoqi Wang","Guodong Zhang","Roger Grosse"],"iclr_id":"SkgsACVKPH","keywords":["generalization","imagenet","optimization","pruning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Picking Winning Tickets Before Training by Preserving Gradient Flow"},"forum":"SkgsACVKPH","id":"SkgsACVKPH"},{"content":{"TLDR":"We present Transformer-XH, which upgrades Transformer with eXtra Hop attentions to intrinsically model structured texts in a data driven way. ","abstract":" ","authors":["Chen Zhao","Chenyan Xiong","Corby Rosset","Xia Song","Paul Bennett","Saurabh Tiwary"],"iclr_id":"r1eIiCNYwS","keywords":["attention","fact verification","multi hop qa","nlp","question answering","reasoning","transformer","verification"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Transformer-XH: Multi-Evidence Reasoning with eXtra Hop Attention"},"forum":"r1eIiCNYwS","id":"r1eIiCNYwS"},{"content":{"TLDR":"We propose to study the problem of few-shot graph classification in graph neural networks (GNNs) to recognize unseen classes, given limited labeled graph examples. Despite several interesting GNN variants being proposed recently for node and graph cl...","abstract":" ","authors":["Jatin Chauhan","Deepak Nathani","Manohar Kaul"],"iclr_id":"Bkeeca4Kvr","keywords":["active learning","fewshot learning","graph networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"FEW-SHOT LEARNING ON GRAPHS VIA SUPER-CLASSES BASED ON GRAPH SPECTRAL MEASURES"},"forum":"Bkeeca4Kvr","id":"Bkeeca4Kvr"},{"content":{"TLDR":"MQL is a simple off-policy meta-RL algorithm that recycles data from the meta-training replay buffer to adapt to new tasks.","abstract":" ","authors":["Rasool Fakoor","Pratik Chaudhari","Stefano Soatto","Alexander J. Smola"],"iclr_id":"SJeD3CEFPH","keywords":["meta reinforcement learning","off policy","reinforcement learning"],"recs":[],"session":["Monday: Meta-learning"],"session_links":[null],"session_times":[null],"title":"Meta-Q-Learning"},"forum":"SJeD3CEFPH","id":"SJeD3CEFPH"},{"content":{"TLDR":"We propose a novel framework for meta-learning a gradient-based update rule that scales to beyond few-shot learning and is applicable to any form of learning, including continual learning.","abstract":" ","authors":["Sebastian Flennerhag","Andrei A. Rusu","Razvan Pascanu","Francesco Visin","Hujun Yin","Raia Hadsell"],"iclr_id":"rkeiQlBFPB","keywords":["gradient descent","inductive bias","meta learning","reinforcement learning","transfer learning"],"recs":[],"session":["Monday: Meta-learning"],"session_links":[null],"session_times":[null],"title":"Meta-Learning with Warped Gradient Descent"},"forum":"rkeiQlBFPB","id":"rkeiQlBFPB"},{"content":{"TLDR":"A simple and effective alternative to adversarial imitation learning: initialize experience replay buffer with demonstrations, set their reward to +1, set reward for all other data to 0, run Q-learning or soft actor-critic to train.","abstract":" ","authors":["Siddharth Reddy","Anca D. Dragan","Sergey Levine"],"iclr_id":"S1xKd24twB","keywords":["adversarial","imitation learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"SQIL: Imitation Learning via Reinforcement Learning with Sparse Rewards"},"forum":"S1xKd24twB","id":"S1xKd24twB"},{"content":{"TLDR":"We present a novel approach for the off-policy estimation problem in infinite-horizon RL.","abstract":" ","authors":["Ali Mousavi","Lihong Li","Qiang Liu","Denny Zhou"],"iclr_id":"S1ltg1rFDS","keywords":["generalization","importance sampling","reinforcement learning","robotics"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Black-box Off-policy Estimation for Infinite-Horizon Reinforcement Learning"},"forum":"S1ltg1rFDS","id":"S1ltg1rFDS"},{"content":{"TLDR":"Several graph problems are impossible unless the product of a graph neural network's depth and width exceeds a polynomial of the graph size.","abstract":" ","authors":["Andreas Loukas"],"iclr_id":"B1l2bp4YwS","keywords":["capacity","distributed","expressive power","graph networks","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"What graph neural networks cannot learn: depth vs width"},"forum":"B1l2bp4YwS","id":"B1l2bp4YwS"},{"content":{"TLDR":"When performing imitation learning from expert demonstrations, distribution matching is a popular approach, in which one alternates between estimating distribution ratios and then using these ratios as rewards in a standard reinforcement learning (RL...","abstract":" ","authors":["Ilya Kostrikov","Ofir Nachum","Jonathan Tompson"],"iclr_id":"Hyg-JC4FDr","keywords":["adversarial","imitation learning","optimization","reinforcement learning","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Imitation Learning via Off-Policy Distribution Matching"},"forum":"Hyg-JC4FDr","id":"Hyg-JC4FDr"},{"content":{"TLDR":"By proposing new visualization techniques we give better insights on GANs optimization in practical settings, we show that GANs on challenging datasets exhibit rotational behavior and do not converge to Nash-Equilibria","abstract":" ","authors":["Hugo Berard","Gauthier Gidel","Amjad Almahairi","Pascal Vincent","Simon Lacoste-Julien"],"iclr_id":"HJeVnCEKwH","keywords":["adversarial","gan","generative models","optimization","visualization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Closer Look at the Optimization Landscapes of Generative Adversarial Networks"},"forum":"HJeVnCEKwH","id":"HJeVnCEKwH"},{"content":{"TLDR":"We present a novel autoregressive video generation that achieves strong results on popular datasets and produces encouraging continuations of real world videos.","abstract":" ","authors":["Dirk Weissenborn","Oscar T\u00e4ckstr\u00f6m","Jakob Uszkoreit"],"iclr_id":"rJgsskrFwH","keywords":["adversarial","attention","autoregressive models","generation","generative models","self attention","video generation","video prediction"],"recs":[],"session":["Monday: Signals and Systems"],"session_links":[null],"session_times":[null],"title":"Scaling Autoregressive Video Models"},"forum":"rJgsskrFwH","id":"rJgsskrFwH"},{"content":{"TLDR":"This paper studies learning the representations of whole graphs in both unsupervised and semi-supervised scenarios. Graph-level representations are critical in a variety of real-world applications such as predicting the properties of molecules and co...","abstract":" ","authors":["Fan-Yun Sun","Jordan Hoffman","Vikas Verma","Jian Tang"],"iclr_id":"r1lfF2NYvH","keywords":["generalization","mutual information","nlp","representation learning","unsupervised"],"recs":[],"session":["Thursday: Graphs and Representations"],"session_links":[null],"session_times":[null],"title":"InfoGraph: Unsupervised and Semi-supervised Graph-Level Representation Learning via Mutual Information Maximization"},"forum":"r1lfF2NYvH","id":"r1lfF2NYvH"},{"content":{"TLDR":"We conduct a comparative study of cross-lingual alignment vs joint training methods and unify these two previously exclusive paradigms in a new framework. ","abstract":" ","authors":["Zirui Wang","Jiateng Xie","Ruochen Xu","Yiming Yang","Graham Neubig","Jaime G. Carbonell"],"iclr_id":"S1l-C0NtwS","keywords":["transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Cross-lingual Alignment vs Joint Training: A Comparative Study and A Simple Unified Framework"},"forum":"S1l-C0NtwS","id":"S1l-C0NtwS"},{"content":{"TLDR":"Learning to extract distinguishable keypoints from a proxy task, outlier rejection.","abstract":" ","authors":["Jiexiong Tang","Hanme Kim","Vitor Guizilini","Sudeep Pillai","Rares Ambrus"],"iclr_id":"Skx82ySYPH","keywords":["self supervised learning","self supervision"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Neural Outlier Rejection for Self-Supervised Keypoint Learning"},"forum":"Skx82ySYPH","id":"Skx82ySYPH"},{"content":{"TLDR":"Training neural networks so you can efficiently patch them later.","abstract":" ","authors":["Anton Sinitsin","Vsevolod Plokhotnyuk","Dmitry Pyrkin","Sergei Popov","Artem Babenko"],"iclr_id":"HJedXaEtvS","keywords":["machine translation","maml","meta learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Editable Neural Networks"},"forum":"HJedXaEtvS","id":"HJedXaEtvS"},{"content":{"TLDR":"Bsuite is a collection of carefully-designed experiments that investigate the core capabilities of RL agents.","abstract":" ","authors":["Ian Osband","Yotam Doron","Matteo Hessel","John Aslanides","Eren Sezener","Andre Saraiva","Katrina McKinney","Tor Lattimore","Csaba Szepesvari","Satinder Singh","Benjamin Van Roy","Richard Sutton","David Silver","Hado Van Hasselt"],"iclr_id":"rygf-kSYwH","keywords":["reinforcement learning","reproducibility","scalability"],"recs":[],"session":["Monday: Reliable RL"],"session_links":[null],"session_times":[null],"title":"Behaviour Suite for Reinforcement Learning"},"forum":"rygf-kSYwH","id":"rygf-kSYwH"},{"content":{"TLDR":"GANs have been applied to text generation and are believed SOTA. However, we propose a new evaluation protocol demonstrating that maximum-likelihood trained models are still better.","abstract":" ","authors":["Massimo Caccia","Lucas Caccia","William Fedus","Hugo Larochelle","Joelle Pineau","Laurent Charlin"],"iclr_id":"BJgza6VtPB","keywords":["adversarial","gan","generation","natural language generation","nlp","text generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Language GANs Falling Short"},"forum":"BJgza6VtPB","id":"BJgza6VtPB"},{"content":{"TLDR":"We introduce an extra data-dependent Gaussian prior objective to augment the current MLE training, which is designed to capture the prior knowledge in the ground-truth data.","abstract":" ","authors":["Zuchao Li","Rui Wang","Kehai Chen","Masso Utiyama","Eiichiro Sumita","Zhuosheng Zhang","Hai Zhao"],"iclr_id":"S1efxTVYDr","keywords":["generation","machine translation","nlp","regularization","unsupervised"],"recs":[],"session":["Thursday: Natural Language"],"session_links":[null],"session_times":[null],"title":"Data-dependent Gaussian Prior Objective for Language Generation"},"forum":"S1efxTVYDr","id":"S1efxTVYDr"},{"content":{"TLDR":"nearly isometric DNN initializations imply low parameter space curvature, and a lower condition number, but that's not always great","abstract":" ","authors":["Piotr Aleksander Sok\u00f3\u0142","Il Memming Park"],"iclr_id":"rkg1ngrFPr","keywords":["gradient descent","information geometry","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Information Geometry of Orthogonal Initializations and Training"},"forum":"rkg1ngrFPr","id":"rkg1ngrFPr"},{"content":{"TLDR":"We propose a novel semi-supervised learning approach with SOTA performance on combating learning with noisy labels.","abstract":" ","authors":["Junnan Li","Richard Socher","Steven C.H. Hoi"],"iclr_id":"HJgExaVtwr","keywords":["semi supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"DivideMix: Learning with Noisy Labels as Semi-supervised Learning"},"forum":"HJgExaVtwr","id":"HJgExaVtwr"},{"content":{"TLDR":"Deep RL policies can be attacked by other agents taking actions so as to create natural observations that are adversarial.","abstract":" ","authors":["Adam Gleave","Michael Dennis","Cody Wild","Neel Kant","Sergey Levine","Stuart Russell"],"iclr_id":"HJgEMpVFwB","keywords":["adversarial","multi agent","perturbation","reinforcement learning","security"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Adversarial Policies: Attacking Deep Reinforcement Learning"},"forum":"HJgEMpVFwB","id":"HJgEMpVFwB"},{"content":{"TLDR":"We present a novel generative model that combines state-of-the-art neural text- to-speech (TTS) with semi-supervised probabilistic latent variable models. By providing partial supervision to some of the latent variables, we are able to force them to ...","abstract":" ","authors":["Raza Habib","Soroosh Mariooryad","Matt Shannon","Eric Battenberg","RJ Skerry-Ryan","Daisy Stanton","David Kao","Tom Bagby"],"iclr_id":"rJeqeCEtvH","keywords":["disentanglement","generative models","unsupervised","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Semi-Supervised Generative Modeling for Controllable Speech Synthesis"},"forum":"rJeqeCEtvH","id":"rJeqeCEtvH"},{"content":{"TLDR":"An approach to learn program policies for control tasks that inductively generalize. ","abstract":" ","authors":["Jeevana Priya Inala","Osbert Bastani","Zenna Tavares","Armando Solar-Lezama"],"iclr_id":"S1l8oANFDH","keywords":["generalization","imitation learning","program synthesis","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Synthesizing Programmatic Policies that Inductively Generalize"},"forum":"S1l8oANFDH","id":"S1l8oANFDH"},{"content":{"TLDR":"Learn how to quantize speech signal and apply algorithms requiring discrete inputs to audio data such as BERT.","abstract":" ","authors":["Alexei Baevski","Steffen Schneider","Michael Auli"],"iclr_id":"rylwJxrYDS","keywords":["clustering","representation learning","self supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"vq-wav2vec: Self-Supervised Learning of Discrete Speech Representations"},"forum":"rylwJxrYDS","id":"rylwJxrYDS"},{"content":{"TLDR":"We investigate the identifiability and interpretability of attention distributions and tokens within contextual embeddings in the self-attention based BERT model.","abstract":" ","authors":["Gino Brunner","Yang Liu","Damian Pascual","Oliver Richter","Massimiliano Ciaramita","Roger Wattenhofer"],"iclr_id":"BJg1f6EFDB","keywords":["attention","generation","interpretability","nlp","self attention","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On Identifiability in Transformers"},"forum":"BJg1f6EFDB","id":"BJg1f6EFDB"},{"content":{"TLDR":"A self-attention layer can perform convolution and often learns to do so in practice.","abstract":" ","authors":["Jean-Baptiste Cordonnier","Andreas Loukas","Martin Jaggi"],"iclr_id":"HJlnC1rKPB","keywords":["attention","capacity","cnn","self attention","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On the Relationship between Self-Attention and Convolutional Layers"},"forum":"HJlnC1rKPB","id":"HJlnC1rKPB"},{"content":{"TLDR":"An efficient differentiable ILP model that learns first-order logic rules that can explain the data.","abstract":" ","authors":["Yuan Yang","Le Song"],"iclr_id":"SJlh8CEYDB","keywords":["attention","interpretability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learn to Explain Efficiently via Neural Logic Inductive Learning"},"forum":"SJlh8CEYDB","id":"SJlh8CEYDB"},{"content":{"TLDR":"Answering a wide class of logical queries over knowledge graphs with box embeddings in vector space","abstract":" ","authors":["Hongyu Ren","Weihua Hu","Jure Leskovec"],"iclr_id":"BJgr4kSFDS","keywords":["graph embedding","knowledge graph embeddings","knowledge graphs","logical reasoning","reasoning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Query2box: Reasoning over Knowledge Graphs in Vector Space Using Box Embeddings"},"forum":"BJgr4kSFDS","id":"BJgr4kSFDS"},{"content":{"TLDR":"We proposed a novel distributed backdoor attack on federated learning and show that it is not only more effective compared with standard centralized attacks, but also harder to be defended by existing robust FL methods","abstract":" ","authors":["Chulin Xie","Keli Huang","Pin-Yu Chen","Bo Li"],"iclr_id":"rkgyS0VFvr","keywords":["adversarial","distributed","federated learning","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"DBA: Distributed Backdoor Attacks against Federated Learning"},"forum":"rkgyS0VFvr","id":"rkgyS0VFvr"},{"content":{"TLDR":"A study on the benefit of sharing representation in Multi-Task Reinforcement Learning.","abstract":" ","authors":["Carlo D'Eramo","Davide Tateo","Andrea Bonarini","Marcello Restelli","Jan Peters"],"iclr_id":"rkgpv2VFvr","keywords":["multi task","reinforcement learning","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Sharing Knowledge in Multi-Task Deep Reinforcement Learning"},"forum":"rkgpv2VFvr","id":"rkgpv2VFvr"},{"content":{"TLDR":"We develop a deep graph matching architecture which refines initial correspondences in order to reach neighborhood consensus.","abstract":" ","authors":["Matthias Fey","Jan E. Lenssen","Christopher Morris","Jonathan Masci","Nils M. Kriege"],"iclr_id":"HyeJf1HKvS","keywords":["computer vision","graph networks","knowledge graphs"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep Graph Matching Consensus"},"forum":"HyeJf1HKvS","id":"HyeJf1HKvS"},{"content":{"TLDR":"we present a principled approach to the problem of federated domain adaptation, which aims to align the representations learned among the different nodes with the data distribution of the target node.","abstract":" ","authors":["Xingchao Peng","Zijun Huang","Yizhe Zhu","Kate Saenko"],"iclr_id":"HJezF3VYPB","keywords":["adversarial","attention","disentanglement","distributed","domain adaptation","federated learning","privacy","text classification","transfer learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Federated Adversarial Domain Adaptation"},"forum":"HJezF3VYPB","id":"HJezF3VYPB"},{"content":{"TLDR":"We propose a neural question requirement inspection model called NeurQuRI that extracts a list of conditions from the question, each of which should be satisfied by the candidate answer generated by an MRC model.","abstract":" ","authors":["Seohyun Back","Sai Chetan Chinthakindi","Akhil Kedia","Haejun Lee","Jaegul Choo"],"iclr_id":"ryxgsCVYPr","keywords":["attention","question answering","reading comprehension"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"NeurQuRI: Neural Question Requirement Inspector for Answerability Prediction in Machine Reading Comprehension"},"forum":"ryxgsCVYPr","id":"ryxgsCVYPr"},{"content":{"TLDR":"We introduce attentive feature distillation and selection, to fine-tune a large model and produce a faster one.","abstract":" ","authors":["Kafeng Wang","Xitong Gao","Yiren Zhao","Xingjian Li","Dejing Dou","Cheng-Zhong Xu"],"iclr_id":"ryxyCeHtPB","keywords":["attention","cnn","imagenet","pruning","regularization","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Pay Attention to Features, Transfer Learn Faster CNNs"},"forum":"ryxyCeHtPB","id":"ryxyCeHtPB"},{"content":{"TLDR":"Deep neural networks achieve remarkable performance in many computer vision tasks. Most state-of-the-art~(SOTA) semantic segmentation and object detection approaches reuse neural network architectures designed for image classification as the backbone...","abstract":" ","authors":["Jiemin Fang","Yuzhu Sun","Kangjian Peng","Qian Zhang","Yuan Li","Wenyu Liu","Xinggang Wang"],"iclr_id":"rklTmyBKPH","keywords":["computer vision","imagenet","neural architecture search","semantic segmentation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Fast Neural Network Adaptation via Parameter Remapping and Architecture Search"},"forum":"rklTmyBKPH","id":"rklTmyBKPH"},{"content":{"TLDR":"The effectiveness of Convolutional Neural Networks (CNNs) has been substantially attributed to their built-in property of translation equivariance. However, CNNs do not have embedded mechanisms to handle other types of transformations. In this work, ...","abstract":" ","authors":["Ivan Sosnovik","Micha\u0142 Szmaja","Arnold Smeulders"],"iclr_id":"HJgpugrKPS","keywords":["attention","cnn","equivariance","stability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Scale-Equivariant Steerable Networks"},"forum":"HJgpugrKPS","id":"HJgpugrKPS"},{"content":{"TLDR":"Interpolations in the latent space demonstrate generalization capacity of GANs and the effect of dataset biases.","abstract":" ","authors":["Ali Jahanian","Lucy Chai","Phillip Isola"],"iclr_id":"HylsTT4FvB","keywords":["adversarial","capacity","gan","generalization","generative models"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On the \"steerability\" of generative adversarial networks"},"forum":"HylsTT4FvB","id":"HylsTT4FvB"},{"content":{"TLDR":"We propose CR-NAS to reallocate engaged  computation resources in different resolution and spatial position.","abstract":" ","authors":["Feng Liang","Chen Lin","Ronghao Guo","Ming Sun","Wei Wu","Junjie Yan","Wanli Ouyang"],"iclr_id":"SkxLFaNKwB","keywords":["neural architecture search","object detection"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Computation Reallocation for Object Detection"},"forum":"SkxLFaNKwB","id":"SkxLFaNKwB"},{"content":{"TLDR":"Gradient clipping provably accelerates gradient descent for non-smooth non-convex functions.","abstract":" ","authors":["Jingzhao Zhang","Tianxing He","Suvrit Sra","Ali Jadbabaie"],"iclr_id":"BJgnXpVYwS","keywords":["gradient descent","optimization"],"recs":[],"session":["Monday: Optimisation Theory"],"session_links":[null],"session_times":[null],"title":"Why Gradient Clipping Accelerates Training: A Theoretical Justification for Adaptivity"},"forum":"BJgnXpVYwS","id":"BJgnXpVYwS"},{"content":{"TLDR":"We study the sparsity-inducing bias of deep models, caused by their learning dynamics.","abstract":" ","authors":["Daniel Gissin","Shai Shalev-Shwartz","Amit Daniely"],"iclr_id":"H1lj0nNFwB","keywords":["generalization","gradient descent","implicit bias","incremental learning","optimization","regularization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"The Implicit Bias of Depth: How Incremental Learning Drives Generalization"},"forum":"H1lj0nNFwB","id":"H1lj0nNFwB"},{"content":{"TLDR":"Learning embedding for control with high-dimensional observations","abstract":" ","authors":["Nir Levine","Yinlam Chow","Rui Shu","Ang Li","Mohammad Ghavamzadeh","Hung Bui"],"iclr_id":"BJxG_0EtDS","keywords":["representation learning","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Prediction, Consistency, Curvature: Representation Learning for Locally-Linear Control"},"forum":"BJxG_0EtDS","id":"BJxG_0EtDS"},{"content":{"TLDR":"Using ensembles and pseudo labels for unsupervised clustering ","abstract":" ","authors":["Divam Gupta","Ramachandran Ramjee","Nipun Kwatra","Muthian Sivathanu"],"iclr_id":"rJlnxkSYPS","keywords":["clustering","ensembles","semi supervised learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Unsupervised Clustering using Pseudo-semi-supervised Learning"},"forum":"rJlnxkSYPS","id":"rJlnxkSYPS"},{"content":{"TLDR":"We propose a generic framework that allows for exploiting the low-rank structure in both planning and deep reinforcement learning.","abstract":" ","authors":["Yuzhe Yang","Guo Zhang","Zhi Xu","Dina Katabi"],"iclr_id":"rklHqRVKvH","keywords":["planning","reinforcement learning"],"recs":[],"session":["Wednesday: RL and Planning"],"session_links":[null],"session_times":[null],"title":"Harnessing Structures for Value-Based Planning and Reinforcement Learning"},"forum":"rklHqRVKvH","id":"rklHqRVKvH"},{"content":{"TLDR":"Sequence model that dynamically adjusts the amount of computation for each input.","abstract":" ","authors":["Maha Elbayad","Jiatao Gu","Edouard Grave","Michael Auli"],"iclr_id":"SJg7KhVKPH","keywords":["capacity","nlp","sequence modeling","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Depth-Adaptive Transformer"},"forum":"SJg7KhVKPH","id":"SJg7KhVKPH"},{"content":{"TLDR":"Graph neural networks (GNNs) have been widely used for representation learning on graph data. However, there is limited understanding on how much performance GNNs actually gain from graph data. This paper introduces a context-surrounding GNN framewor...","abstract":" ","authors":["Yifan Hou","Jian Zhang","James Cheng","Kaili Ma","Richard T. B. Ma","Hongzhi Chen","Ming-Chang Yang"],"iclr_id":"rkeIIkHKvS","keywords":["graph networks","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Measuring and Improving the Use of Graph Information in Graph Neural Networks"},"forum":"rkeIIkHKvS","id":"rkeIIkHKvS"},{"content":{"TLDR":"It has recently been shown that ReLU networks produce arbitrarily over-confident predictions far away from the \ntraining data. Thus, ReLU networks do not know when they don't know. However, this is a highly important property in safety\ncritical appli...","abstract":" ","authors":["Alexander Meinke","Matthias Hein"],"iclr_id":"ByxGkySKwH","keywords":["relu networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Towards neural networks that provably know when they don't know"},"forum":"ByxGkySKwH","id":"ByxGkySKwH"},{"content":{"TLDR":"We study the implicit bias of gradient descent and prove under a minimal set of assumptions that the parameter direction of homogeneous models converges to KKT points of a natural margin maximization problem.","abstract":" ","authors":["Kaifeng Lyu","Jian Li"],"iclr_id":"SJeLIgBKPS","keywords":["cnn","gradient descent","optimization","regression","regularization","robustness"],"recs":[],"session":["Wednesday: Theory"],"session_links":[null],"session_times":[null],"title":"Gradient Descent Maximizes the Margin of Homogeneous Neural Networks"},"forum":"SJeLIgBKPS","id":"SJeLIgBKPS"},{"content":{"TLDR":"A simple and effective method for reducing large neural networks to flexible parameter targets based on block substitution.","abstract":" ","authors":["Jack Turner","Elliot J. Crowley","Michael O'Boyle","Amos Storkey","Gavin Gray"],"iclr_id":"SklkDkSFPB","keywords":["capacity","cnn","compression","imagenet","model compression","network compression","neural architecture search"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"BlockSwap: Fisher-guided Block Substitution for Network Compression on a Budget"},"forum":"SklkDkSFPB","id":"SklkDkSFPB"},{"content":{"TLDR":"In this paper, we proposed a novel algorithm, GenDICE, for general stationary distribution correction estimation, which can handle both discounted and average off-policy evaluation on multiple behavior-agnostic samples.","abstract":" ","authors":["Ruiyi Zhang","Bo Dai","Lihong Li","Dale Schuurmans"],"iclr_id":"HkxlcnVFwB","keywords":["off policy policy evaluation","reinforcement learning"],"recs":[],"session":["Tuesday: RL and Estimation"],"session_links":[null],"session_times":[null],"title":"GenDICE: Generalized Offline Estimation of Stationary Values"},"forum":"HkxlcnVFwB","id":"HkxlcnVFwB"},{"content":{"TLDR":"RGBD image generation for unsupervised camera parameter conditioning","abstract":" ","authors":["Atsuhiro Noguchi","Tatsuya Harada"],"iclr_id":"HyxjNyrtPr","keywords":["generation","generative models","image generation","representation learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"RGBD-GAN: Unsupervised 3D Representation Learning From Natural Image Datasets via RGBD Image Synthesis"},"forum":"HyxjNyrtPr","id":"HyxjNyrtPr"},{"content":{"TLDR":"We propose a transductive meta-learning algorithm using synthetic gradients, analyze its generalization via information bottleneck, show SOTA results on few-shot learning.","abstract":" ","authors":["Shell Xu Hu","Pablo Moreno","Yang Xiao","Xi Shen","Guillaume Obozinski","Neil Lawrence","Andreas Damianou"],"iclr_id":"Hkg-xgrYvH","keywords":["gradient descent","imagenet","information bottleneck","meta learning","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Empirical Bayes Transductive Meta-Learning with Synthetic Gradients"},"forum":"Hkg-xgrYvH","id":"Hkg-xgrYvH"},{"content":{"TLDR":"This paper presents a phenomenon in neural networks that we refer to as local elasticity. Roughly speaking, a classifier is said to be locally elastic if its prediction at a feature vector x' is not significantly perturbed, after the classifier is up...","abstract":" ","authors":["Hangfeng He","Weijie Su"],"iclr_id":"HJxMYANtPH","keywords":["clustering","gradient descent","neural tangent kernel"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"The Local Elasticity of Neural Networks"},"forum":"HJxMYANtPH","id":"HJxMYANtPH"},{"content":{"TLDR":"The posteriors over neural network weights are high dimensional and multimodal. Each mode typically characterizes a meaningfully different representation of the data. We develop Cyclical Stochastic Gradient MCMC (SG-MCMC) to automatically explore suc...","abstract":" ","authors":["Ruqi Zhang","Chunyuan Li","Jianyi Zhang","Changyou Chen","Andrew Gordon Wilson"],"iclr_id":"rkeS1RVtPS","keywords":["bayesian inference","imagenet"],"recs":[],"session":["Tuesday: Probabilistic Approaches"],"session_links":[null],"session_times":[null],"title":"Cyclical Stochastic Gradient MCMC for Bayesian Deep Learning"},"forum":"rkeS1RVtPS","id":"rkeS1RVtPS"},{"content":{"TLDR":"We introduce the Continuous Logic Network (CLN), a novel neural architecture for automatically learning loop invariants and general SMT formulas.","abstract":" ","authors":["Gabriel Ryan","Justin Wong","Jianan Yao","Ronghui Gu","Suman Jana"],"iclr_id":"HJlfuTEtvB","keywords":["verification"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"CLN2INV: Learning Loop Invariants with Continuous Logic Networks"},"forum":"HJlfuTEtvB","id":"HJlfuTEtvB"},{"content":{"TLDR":"We extend deep sets to functional embeddings and Neural Processes to include translation equivariant members","abstract":" ","authors":["Jonathan Gordon","Wessel P. Bruinsma","Andrew Y. K. Foong","James Requeima","Yann Dubois","Richard E. Turner"],"iclr_id":"Skey4eBYPS","keywords":["equivariance","generalization","inductive bias"],"recs":[],"session":["Tuesday: Feature Discovery for Structured Data"],"session_links":[null],"session_times":[null],"title":"Convolutional Conditional Neural Processes"},"forum":"Skey4eBYPS","id":"Skey4eBYPS"},{"content":{"TLDR":"Automatic differentiation frameworks are optimized for exactly one thing: computing the average mini-batch gradient. Yet, other quantities such as the variance of the mini-batch gradients or many approximations to the Hessian can, in theory, be compu...","abstract":" ","authors":["Felix Dangel","Frederik Kunstner","Philipp Hennig"],"iclr_id":"BJlrF24twB","keywords":["optimization"],"recs":[],"session":["Thursday: Fairness, Interpretabiity and Deployment"],"session_links":[null],"session_times":[null],"title":"BackPACK: Packing more into Backprop"},"forum":"BJlrF24twB","id":"BJlrF24twB"},{"content":{"TLDR":"We systematically analyze the convergence of popular gradient algorithms for solving bilinear games, with both simultaneous and alternating updates.","abstract":" ","authors":["Guojun Zhang","Yaoliang Yu"],"iclr_id":"SJlVY04FwH","keywords":["adversarial","attention","gan","generative models","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Convergence of Gradient Methods on Bilinear Zero-Sum Games"},"forum":"SJlVY04FwH","id":"SJlVY04FwH"},{"content":{"TLDR":"Language modeling for lifelong language learning.","abstract":" ","authors":["Fan-Keng Sun","Cheng-Hao Ho","Hung-Yi Lee"],"iclr_id":"Skgxcn4YDS","keywords":["capacity","catastrophic forgetting","language modeling","lifelong learning","memory","nlp"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"LAMOL: LAnguage MOdeling for Lifelong Language Learning"},"forum":"Skgxcn4YDS","id":"Skgxcn4YDS"},{"content":{"TLDR":"We proposed a deep graph matching method with novel channel-independent embedding and Hungarian loss, which achieved state-of-the-art performance.","abstract":" ","authors":["Tianshu Yu","Runzhong Wang","Junchi Yan","Baoxin Li"],"iclr_id":"rJgBd2NYPH","keywords":["attention"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning deep graph matching with channel-independent embedding and Hungarian attention"},"forum":"rJgBd2NYPH","id":"rJgBd2NYPH"},{"content":{"TLDR":"Reinforcement learning encounters major challenges in multi-agent settings, such as scalability and non-stationarity. Recently, value function factorization learning emerges as a promising way to address these challenges in collaborative multi-agent ...","abstract":" ","authors":["Tonghan Wang","Jianhao Wang","Chongyi Zheng","Chongjie Zhang"],"iclr_id":"HJx-3grYDB","keywords":["multi agent reinforcement learning","mutual information","reinforcement learning","scalability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning Nearly Decomposable Value Functions Via Communication Minimization"},"forum":"HJx-3grYDB","id":"HJx-3grYDB"},{"content":{"TLDR":"The paper describes a flexible framework for building CNNs that are equivariant to a large class of transformations groups.","abstract":" ","authors":["Erik J Bekkers"],"iclr_id":"H1gBhkBFDH","keywords":["cnn","computer vision","equivariance"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"B-Spline CNNs on Lie groups"},"forum":"H1gBhkBFDH","id":"H1gBhkBFDH"},{"content":{"TLDR":"We introduced BatchEnsemble, an efficient method for ensembling and lifelong learning which can be used to improve the accuracy and uncertainty of any neural network like typical ensemble methods.","abstract":" ","authors":["Yeming Wen","Dustin Tran","Jimmy Ba"],"iclr_id":"Sklf1yrYDr","keywords":["ensembles","imagenet","lifelong learning","memory","uncertainty"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"BatchEnsemble: an Alternative Approach to Efficient Ensemble and Lifelong Learning"},"forum":"Sklf1yrYDr","id":"Sklf1yrYDr"},{"content":{"TLDR":"Transductive fine-tuning of a deep network is a strong baseline for few-shot image classification and outperforms the state-of-the-art on all standard benchmarks.","abstract":" ","authors":["Guneet Singh Dhillon","Pratik Chaudhari","Avinash Ravichandran","Stefano Soatto"],"iclr_id":"rylXBkrYDS","keywords":["fewshot learning","fine tuning","imagenet","meta learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Baseline for Few-Shot Image Classification"},"forum":"rylXBkrYDS","id":"rylXBkrYDS"},{"content":{"TLDR":"Representation/knowledge distillation by maximizing mutual information between teacher and student","abstract":" ","authors":["Yonglong Tian","Dilip Krishnan","Phillip Isola"],"iclr_id":"SkgpBJrtvS","keywords":["compression","knowledge distillation","model compression","mutual information","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Contrastive Representation Distillation"},"forum":"SkgpBJrtvS","id":"SkgpBJrtvS"},{"content":{"TLDR":"MMD-based, saddle-point optimisation free, stable-to-train generative model that beats GAN on generative quality without playing any  zero-sum games.","abstract":" ","authors":["Akash Srivastava","Kai Xu","Michael U. Gutmann","Charles Sutton"],"iclr_id":"SJg7spEYDS","keywords":["adversarial","generative models","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Generative Ratio Matching Networks"},"forum":"SJg7spEYDS","id":"SJg7spEYDS"},{"content":{"TLDR":"Neural nets can encode complex 3D objects into the parameters of other (surprisingly small) neural nets","abstract":" ","authors":["Eric Mitchell","Selim Engin","Volkan Isler","Daniel D Lee"],"iclr_id":"HJgfDREKDB","keywords":["3d reconstruction","computer vision","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Higher-Order Function Networks for Learning Composable 3D Object Representations"},"forum":"HJgfDREKDB","id":"HJgfDREKDB"},{"content":{"TLDR":"We demonstrate the utility of a recent AI explainability technique by visualizing the learned features of a CNN trained on binary classification of zebrafish movements.","abstract":" ","authors":["Bennet Breier","Arno Onken"],"iclr_id":"rJgQkT4twH","keywords":["attention","cnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Analysis of Video Feature Learning in Two-Stream CNNs on the Example of Zebrafish Swim Bout Classification"},"forum":"rJgQkT4twH","id":"rJgQkT4twH"},{"content":{"TLDR":"The paper designs two algorithms for the stochastic AUC maximization problem with state-of-the-art complexities when using deep neural network as predictive model, which are also verified by empirical studies.","abstract":" ","authors":["Mingrui Liu","Zhuoning Yuan","Yiming Ying","Tianbao Yang"],"iclr_id":"HJepXaVYDr","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Stochastic AUC Maximization with Deep Neural Networks"},"forum":"HJepXaVYDr","id":"HJepXaVYDr"},{"content":{"TLDR":"We empirically disprove a fundamental hypothesis of the widely-adopted weight sharing strategy in neural architecture search and explain why the state-of-the-arts NAS algorithms performs similarly to random search.","abstract":" ","authors":["Kaicheng Yu","Christian Sciuto","Martin Jaggi","Claudiu Musat","Mathieu Salzmann"],"iclr_id":"H1loF2NFwr","keywords":["neural architecture search","search"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Evaluating The Search Phase of Neural Architecture Search"},"forum":"H1loF2NFwr","id":"H1loF2NFwr"},{"content":{"TLDR":"Sample efficiency has been one of the major challenges for deep reinforcement learning. Non-parametric episodic control has been proposed to speed up parametric reinforcement learning by rapidly latching on previously successful policies. However, pr...","abstract":" ","authors":["Guangxiang Zhu","Zichuan Lin","Guangwen Yang","Chongjie Zhang"],"iclr_id":"HkxjqxBYDB","keywords":["associative memory","episodic memory","memory","navigation","reasoning","reinforcement learning","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Episodic Reinforcement Learning with Associative Memory"},"forum":"HkxjqxBYDB","id":"HkxjqxBYDB"},{"content":{"TLDR":"Improving the sample efficiency in reinforcement learning has been a long-standing research problem. In this work, we aim to reduce the sample complexity of existing policy gradient methods. We propose a novel policy gradient algorithm called SRVR-PG...","abstract":" ","authors":["Pan Xu","Felicia Gao","Quanquan Gu"],"iclr_id":"HJlxIJBFDr","keywords":["policy gradient","reinforcement learning","sample efficiency","variance reduction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Sample Efficient Policy Gradient Methods with Recursive Variance Reduction"},"forum":"HJlxIJBFDr","id":"HJlxIJBFDr"},{"content":{"TLDR":"SlowMo improves the optimization and generalization performance of communication-efficient decentralized algorithms without sacrificing speed.","abstract":" ","authors":["Jianyu Wang","Vinayak Tantia","Nicolas Ballas","Michael Rabbat"],"iclr_id":"SkxJ8REYPH","keywords":["distributed","distributed optimization","generalization","machine translation","momentum","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"SlowMo: Improving Communication-Efficient Distributed SGD with Slow Momentum"},"forum":"SkxJ8REYPH","id":"SkxJ8REYPH"},{"content":{"TLDR":"We propose a new DNN architecture for deep learning on tabular data","abstract":" ","authors":["Sergei Popov","Stanislav Morozov","Artem Babenko"],"iclr_id":"r1eiu2VtwH","keywords":["dnn","ensembles","optimization","representation learning","tabular data"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Neural Oblivious Decision Ensembles for Deep Learning on Tabular Data"},"forum":"r1eiu2VtwH","id":"r1eiu2VtwH"},{"content":{"TLDR":"We learn appliance usage patterns in homes without labels, using self-supervised learning with energy and location data","abstract":" ","authors":["Chen-Yu Hsu","Abbas Zeitoun","Guang-He Lee","Dina Katabi","Tommi Jaakkola"],"iclr_id":"B1lJzyStvS","keywords":["multi modal learning","self supervised learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Self-Supervised Learning of Appliance Usage"},"forum":"B1lJzyStvS","id":"B1lJzyStvS"},{"content":{"TLDR":"We adapt Q-learning with UCB-exploration bonus to infinite-horizon MDP with discounted rewards without accessing a generative model, and improves the previously best known result.","abstract":" ","authors":["Yuanhao Wang","Kefan Dong","Xiaoyu Chen","Liwei Wang"],"iclr_id":"BkglSTNFDB","keywords":["generative models","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Q-learning with UCB Exploration is Sample Efficient for Infinite-Horizon MDP"},"forum":"BkglSTNFDB","id":"BkglSTNFDB"},{"content":{"TLDR":"A method to train a single model simultaneously minimizing a family of loss functions instead of training a set of per-loss models.","abstract":" ","authors":["Alexey Dosovitskiy","Josip Djolonga"],"iclr_id":"HyxY6JHKwr","keywords":["compression","generation","image generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"You Only Train Once: Loss-Conditional Training of Deep Networks"},"forum":"HyxY6JHKwr","id":"HyxY6JHKwr"},{"content":{"TLDR":"We learn a diversity sampling function with DPPs to obtain a diverse set of samples from a generative model.","abstract":" ","authors":["Ye Yuan","Kris M. Kitani"],"iclr_id":"ryxnY3NYPS","keywords":["autoencoder","generative models","gradient descent","perturbation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Diverse Trajectory Forecasting with Determinantal Point Processes"},"forum":"ryxnY3NYPS","id":"ryxnY3NYPS"},{"content":{"TLDR":"Responding with knowledge has been recognized as an important capability for an intelligent conversational agent. Yet knowledge-grounded dialogues, as training data for learning such a response generation model, are difficult to obtain. Motivated by ...","abstract":" ","authors":["Xueliang Zhao","Wei Wu","Chongyang Tao","Can Xu","Dongyan Zhao","Rui Yan"],"iclr_id":"rJeIcTNtvS","keywords":["generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Low-Resource Knowledge-Grounded Dialogue Generation"},"forum":"rJeIcTNtvS","id":"rJeIcTNtvS"},{"content":{"TLDR":"Proposed methods to extract and leverage interpretations of feature interactions","abstract":" ","authors":["Michael Tsang","Dehua Cheng","Hanpeng Liu","Xue Feng","Eric Zhou","Yan Liu"],"iclr_id":"BkgnhTEtDS","keywords":["automl","black box","interpretability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Feature Interaction Interpretability: A Case for Explaining Ad-Recommendation Systems via Neural Interaction Detection"},"forum":"BkgnhTEtDS","id":"BkgnhTEtDS"},{"content":{"TLDR":"We proposed a Nesterov Iterative Fast Gradient Sign Method (NI-FGSM) and a Scale-Invariant attack Method (SIM) that can boost the transferability of adversarial examples for image classification.","abstract":" ","authors":["Jiadong Lin","Chuanbiao Song","Kun He","Liwei Wang","John E. Hopcroft"],"iclr_id":"SJlHwkBYDH","keywords":["adversarial","adversarial attacks","generation","imagenet","optimization","overfitting","perturbation","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Nesterov Accelerated Gradient and Scale Invariance for Adversarial Attacks"},"forum":"SJlHwkBYDH","id":"SJlHwkBYDH"},{"content":{"TLDR":"Learn in temporal point processes by modeling the conditional density, not the conditional intensity.","abstract":" ","authors":["Oleksandr Shchur","Marin Bilo\u0161","Stephan G\u00fcnnemann"],"iclr_id":"HygOjhEYDH","keywords":["missing data"],"recs":[],"session":["Tuesday: Probabilistic Approaches"],"session_links":[null],"session_times":[null],"title":"Intensity-Free Learning of Temporal Point Processes"},"forum":"HygOjhEYDH","id":"HygOjhEYDH"},{"content":{"TLDR":"A Composition-based Graph Convolutional framework for multi-relational graphs.","abstract":" ","authors":["Shikhar Vashishth","Soumya Sanyal","Vikram Nitin","Partha Talukdar"],"iclr_id":"BylA_C4tPr","keywords":["cnn","graph embedding","graph networks","knowledge graph embeddings"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Composition-based Multi-Relational Graph Convolutional Networks"},"forum":"BylA_C4tPr","id":"BylA_C4tPr"},{"content":{"TLDR":"System to learn robotic tasks in the real world with reinforcement learning without instrumentation","abstract":" ","authors":["Henry Zhu","Justin Yu","Abhishek Gupta","Dhruv Shah","Kristian Hartikainen","Avi Singh","Vikash Kumar","Sergey Levine"],"iclr_id":"rJe2syrtvS","keywords":["reinforcement learning","robotics"],"recs":[],"session":["Monday: Reliable RL"],"session_links":[null],"session_times":[null],"title":"The Ingredients of Real World Robotic Reinforcement Learning"},"forum":"rJe2syrtvS","id":"rJe2syrtvS"},{"content":{"TLDR":"We study a multi-layer generalization of the magnitude-based pruning.","abstract":" ","authors":["Sejun Park","Jaeho Lee","Sangwoo Mo","Jinwoo Shin"],"iclr_id":"ryl3ygHYDB","keywords":["optimization","pruning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Lookahead: A Far-sighted Alternative of Magnitude-based Pruning"},"forum":"ryl3ygHYDB","id":"ryl3ygHYDB"},{"content":{"TLDR":"We call into question commonly held beliefs regarding the loss landscape, optimization, network width, and rank.","abstract":" ","authors":["Micah Goldblum","Jonas Geiping","Avi Schwarzschild","Michael Moeller","Tom Goldstein"],"iclr_id":"HyxyIgHFvr","keywords":["batch normalization","deep learning theory","generalization","loss landscape","neural tangent kernel","robustness"],"recs":[],"session":["Wednesday: Theory"],"session_links":[null],"session_times":[null],"title":"Truth or backpropaganda? An empirical investigation of deep learning theory"},"forum":"HyxyIgHFvr","id":"HyxyIgHFvr"},{"content":{"TLDR":"Coupled rule-exemplar supervision and a implication loss helps to jointly learn to denoise rules and imply labels.","abstract":" ","authors":["Abhijeet Awasthi","Sabyasachi Ghosh","Rasna Goyal","Sunita Sarawagi"],"iclr_id":"SkeuexBtDr","keywords":["denoising","weakly supervised learning","denoising","noisy labels","nlp","weakly supervised learning","semi supervised learning","text classification"],"recs":[],"session":["Wednesday: Symbols and Discovery"],"session_links":[null],"session_times":[null],"title":"Learning from Rules Generalizing Labeled Exemplars"},"forum":"SkeuexBtDr","id":"SkeuexBtDr"},{"content":{"TLDR":"We propose a novel learning to branch framework using graph neural networks to improve branch and bound based neural network verification methods. ","abstract":" ","authors":["Jingyue Lu","M. Pawan Kumar"],"iclr_id":"B1evfa4tPB","keywords":["cnn","graph networks","transfer learning","verification"],"recs":[],"session":["Monday: Robustness and Verification"],"session_links":[null],"session_times":[null],"title":"Neural Network Branching for Neural Network Verification "},"forum":"B1evfa4tPB","id":"B1evfa4tPB"},{"content":{"TLDR":"We show that with the right loss and architecture, view-predictive learning improves 3D object detection","abstract":" ","authors":["Adam W. Harley","Shrinidhi K. Lakshmikanth","Fangyu Li","Xian Zhou","Hsiao-Yu Fish Tung","Katerina Fragkiadaki"],"iclr_id":"BJxt60VtPr","keywords":["object discovery","regression","semi supervised learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning from Unlabelled Videos Using Contrastive Predictive Neural 3D Mapping"},"forum":"BJxt60VtPr","id":"BJxt60VtPr"},{"content":{"TLDR":"Graph convolutional networks (GCNs) have achieved remarkable performance in a variety of network science learning tasks. However, theoretical analysis of such approaches is still at its infancy. Graph scattering transforms (GSTs) are non-trainable de...","abstract":" ","authors":["Vassilis N. Ioannidis","Siheng Chen","Georgios B. Giannakis"],"iclr_id":"rJeg7TEYwB","keywords":["cnn","generalization","graph networks","perturbation","pruning","stability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Pruned Graph Scattering Transforms"},"forum":"rJeg7TEYwB","id":"rJeg7TEYwB"},{"content":{"TLDR":"Counterfactual regret minimization (CFR) methods are effective for solving two-player zero-sum extensive games with imperfect information with  state-of-the-art results.  However,  the vanilla CFR has to traverse the whole game tree in each round, wh...","abstract":" ","authors":["Yichi Zhou","Tongzheng Ren","Jialian Li","Dong Yan","Jun Zhu"],"iclr_id":"rJx4p3NYDB","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Lazy-CFR: fast and near-optimal regret minimization for extensive games with imperfect information"},"forum":"rJx4p3NYDB","id":"rJx4p3NYDB"},{"content":{"TLDR":"The NTK linearization is a universal approximator, even when looking arbitrarily close to initialization","abstract":" ","authors":["Ziwei Ji","Matus Telgarsky","Ruicheng Xian"],"iclr_id":"HklQYxBKwS","keywords":["neural tangent kernel","relu networks","universal approximation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Neural tangent kernels, transportation mappings, and universal approximation"},"forum":"HklQYxBKwS","id":"HklQYxBKwS"},{"content":{"TLDR":"A new distributed, asynchronous, SGD-based algorithm, which achieves state-of-the-art accuracy on existing architectures using staleness penalization without having to re-tune the hyperparameters.","abstract":" ","authors":["Saar Barkai","Ido Hakimi","Assaf Schuster"],"iclr_id":"B1lLw6EYwB","keywords":["distributed","gradient descent","imagenet","momentum","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Gap-Aware Mitigation of Gradient Staleness"},"forum":"B1lLw6EYwB","id":"B1lLw6EYwB"},{"content":{"TLDR":"A novel neural architecture which implicitly learns an (oblique) decision tree.","abstract":" ","authors":["Guang-He Lee","Tommi S. Jaakkola"],"iclr_id":"Bke8UR4FPB","keywords":["regression","relu networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Locally Constant Networks"},"forum":"Bke8UR4FPB","id":"Bke8UR4FPB"},{"content":{"TLDR":"A modular method for fully cooperative multi-goal multi-agent reinforcement learning, based on curriculum learning for efficient exploration and credit assignment for action-goal interactions.","abstract":" ","authors":["Jiachen Yang","Alireza Nakhaei","David Isele","Kikuo Fujimura","Hongyuan Zha"],"iclr_id":"S1lEX04tPr","keywords":["multi agent reinforcement learning","navigation","policy gradient","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"CM3: Cooperative Multi-goal Multi-stage Multi-agent Reinforcement Learning"},"forum":"S1lEX04tPr","id":"S1lEX04tPr"},{"content":{"TLDR":"We propose measurement of phrase importance and algorithms for hierarchical explanation of neural sequence model predictions","abstract":" ","authors":["Xisen Jin","Zhongyu Wei","Junyi Du","Xiangyang Xue","Xiang Ren"],"iclr_id":"BkxRRkSKwr","keywords":["interpretability","nlp","transformer"],"recs":[],"session":["Thursday: Natural Language"],"session_links":[null],"session_times":[null],"title":"Towards Hierarchical Importance Attribution: Explaining Compositional Semantics for Neural Sequence Models"},"forum":"BkxRRkSKwr","id":"BkxRRkSKwr"},{"content":{"TLDR":"Meta-learning curiosity algorithms by searching through a rich space of programs yields novel designs that generalize across very different reinforcement-learning domains.","abstract":" ","authors":["Ferran Alet","Martin F. Schneider","Tomas Lozano-Perez","Leslie Pack Kaelbling"],"iclr_id":"BygdyxHFDS","keywords":["curiosity","generalization","meta learning","navigation","nlp","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Meta-learning curiosity algorithms"},"forum":"BygdyxHFDS","id":"BygdyxHFDS"},{"content":{"TLDR":"We propose a differentiable family of \"kaleidoscope matrices,\" prove that all structured matrices can be represented in this form, and use them to replace hand-crafted linear maps in deep learning models.","abstract":" ","authors":["Tri Dao","Nimit Sohoni","Albert Gu","Matthew Eichhorn","Amit Blonder","Megan Leszczynski","Atri Rudra","Christopher R\u00e9"],"iclr_id":"BkgrBgSYDS","keywords":["imagenet","memory","nlp","transformer"],"recs":[],"session":["Tuesday: Representation Learning"],"session_links":[null],"session_times":[null],"title":"Kaleidoscope: An Efficient, Learnable Representation For All Structured Linear Maps"},"forum":"BkgrBgSYDS","id":"BkgrBgSYDS"},{"content":{"TLDR":"Recent Transformer-based models such as Transformer-XL and BERT have achieved huge success on various natural language processing tasks. However, contextualized embeddings at the output layer of these powerful models tend to degenerate and occupy an ...","abstract":" ","authors":["Lingxiao Wang","Jing Huang","Kevin Huang","Ziniu Hu","Guangtao Wang","Quanquan Gu"],"iclr_id":"ByxY8CNtvr","keywords":["generation","machine translation","nlp","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Improving Neural Language Generation with Spectrum Control"},"forum":"ByxY8CNtvr","id":"ByxY8CNtvr"},{"content":{"TLDR":"A DL model for RNA secondary structure prediction, which uses an unrolled algorithm in the architecture to enforce constraints.","abstract":" ","authors":["Xinshi Chen","Yu Li","Ramzan Umarov","Xin Gao","Le Song"],"iclr_id":"S1eALyrYDH","keywords":[],"recs":[],"session":["Tuesday: Biology and ML"],"session_links":[null],"session_times":[null],"title":"RNA Secondary Structure Prediction By Learning Unrolled Algorithms"},"forum":"S1eALyrYDH","id":"S1eALyrYDH"},{"content":{"TLDR":"Single episode policy transfer in a family of environments with related dynamics, via optimized probing for rapid inference of latent variables and immediate execution of a universal policy.","abstract":" ","authors":["Jiachen Yang","Brenden Petersen","Hongyuan Zha","Daniel Faissol"],"iclr_id":"rJeQoCNYDS","keywords":["reinforcement learning","transfer learning","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Single Episode Policy Transfer in Reinforcement Learning"},"forum":"rJeQoCNYDS","id":"rJeQoCNYDS"},{"content":{"TLDR":"We study the certified robustness for top-k predictions via randomized smoothing under Gaussian noise and derive a tight robustness bound in L_2 norm.","abstract":" ","authors":["Jinyuan Jia","Xiaoyu Cao","Binghui Wang","Neil Zhenqiang Gong"],"iclr_id":"BkeWw6VFwr","keywords":["adversarial","imagenet","perturbation","randomized smoothing","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Certified Robustness for Top-k Predictions against Adversarial Perturbations via Randomized Smoothing"},"forum":"BkeWw6VFwr","id":"BkeWw6VFwr"},{"content":{"TLDR":"With the recent success and popularity of pre-trained language models (LMs) in natural language processing, there has been a rise in efforts to understand their inner workings. \nIn line with such interest, we propose a novel method that assists us in...","abstract":" ","authors":["Taeuk Kim","Jihun Choi","Daniel Edmiston","Sang-goo Lee"],"iclr_id":"H1xPR3NtPB","keywords":["nlp"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Are Pre-trained Language Models Aware of Phrases? Simple but Strong Baselines for Grammar Induction"},"forum":"H1xPR3NtPB","id":"H1xPR3NtPB"},{"content":{"TLDR":"This paper provides a rigorous study of the variance reduced TD learning and characterizes its advantage over vanilla TD learning","abstract":" ","authors":["Tengyu Xu","Zhe Wang","Yi Zhou","Yingbin Liang"],"iclr_id":"S1ly10EKDS","keywords":["optimization","reinforcement learning","variance reduction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Reanalysis of Variance Reduced Temporal Difference Learning"},"forum":"S1ly10EKDS","id":"S1ly10EKDS"},{"content":{"TLDR":"DeepV2D predicts depth from a video clip by composing elements of classical SfM into a fully differentiable network.","abstract":" ","authors":["Zachary Teed","Jia Deng"],"iclr_id":"HJeO7RNKPr","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"DeepV2D: Video to Depth with Differentiable Structure from Motion"},"forum":"HJeO7RNKPr","id":"HJeO7RNKPr"},{"content":{"TLDR":"Under certain condition on the input and output linear transformations, both GD and SGD can achieve global convergence for training deep linear ResNets.","abstract":" ","authors":["Difan Zou","Philip M. Long","Quanquan Gu"],"iclr_id":"HJxEhREKDH","keywords":["gradient descent"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On the Global Convergence  of Training Deep Linear ResNets"},"forum":"HJxEhREKDH","id":"HJxEhREKDH"},{"content":{"TLDR":"We use gradient descent on a regularized autoencoder loss to correct anomalous images.","abstract":" ","authors":["David Dehaene","Oriel Frigo","S\u00e9bastien Combrexelle","Pierre Eline"],"iclr_id":"HJx81ySKwr","keywords":["anomaly detection","autoencoder","gradient descent","regularization","unsupervised","unsupervised anomaly detection","variational autoencoders"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Iterative energy-based projection on a normal data manifold for anomaly localization"},"forum":"HJx81ySKwr","id":"HJx81ySKwr"},{"content":{"TLDR":"The success of recent mutual information (MI)-based representation learning approaches strongly depends on the inductive bias in both the choice of network architectures and the parametrization of the employed MI estimators.","abstract":" ","authors":["Michael Tschannen","Josip Djolonga","Paul K. Rubenstein","Sylvain Gelly","Mario Lucic"],"iclr_id":"rkxoh24FPH","keywords":["inductive bias","metric learning","mutual information","representation learning","self supervised learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On Mutual Information Maximization for Representation Learning"},"forum":"rkxoh24FPH","id":"rkxoh24FPH"},{"content":{"TLDR":"We investigate automatic curriculum generation and identify a number of losses useful to learn to generate a curriculum of tasks.","abstract":" ","authors":["Sebastien Racaniere","Andrew Lampinen","Adam Santoro","David Reichert","Vlad Firoiu","Timothy Lillicrap"],"iclr_id":"H1e0Wp4KvH","keywords":["generation","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Automated curriculum generation through setter-solver interactions"},"forum":"H1e0Wp4KvH","id":"H1e0Wp4KvH"},{"content":{"TLDR":"Hypermodels can encode posterior distributions similar to large ensembles at much smaller computational cost. This can facilitate significant improvements in exploration.","abstract":" ","authors":["Vikranth Dwaracherla","Xiuyuan Lu","Morteza Ibrahimi","Ian Osband","Zheng Wen","Benjamin Van Roy"],"iclr_id":"ryx6WgStPB","keywords":["ensembles","hypernetworks","reinforcement learning","uncertainty"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Hypermodels for Exploration"},"forum":"ryx6WgStPB","id":"ryx6WgStPB"},{"content":{"TLDR":"We propose new tensor decompositions and associated regularizers to obtain state of the art performances on temporal knowledge base completion.","abstract":" ","authors":["Timoth\u00e9e Lacroix","Guillaume Obozinski","Nicolas Usunier"],"iclr_id":"rke2P1BFwS","keywords":["knowledge base completion","regularization","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Tensor Decompositions for Temporal Knowledge Base Completion"},"forum":"rke2P1BFwS","id":"rke2P1BFwS"},{"content":{"TLDR":"We introduce a new batch active learning algorithm that's robust to model architecture, batch size, and dataset.","abstract":" ","authors":["Jordan T. Ash","Chicheng Zhang","Akshay Krishnamurthy","John Langford","Alekh Agarwal"],"iclr_id":"ryghZJBKPS","keywords":["active learning","uncertainty"],"recs":[],"session":["Wednesday: Optimisation I"],"session_links":[null],"session_times":[null],"title":"Deep Batch Active Learning by Diverse, Uncertain Gradient Lower Bounds"},"forum":"ryghZJBKPS","id":"ryghZJBKPS"},{"content":{"TLDR":"We demonstrate that flow-based generative models offer a viable and competitive approach to generative modeling of video.","abstract":" ","authors":["Manoj Kumar","Mohammad Babaeizadeh","Dumitru Erhan","Chelsea Finn","Sergey Levine","Laurent Dinh","Durk Kingma"],"iclr_id":"rJgUfTEYvH","keywords":["autoregressive models","generation","generative models","optimization","video generation","video prediction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"VideoFlow: A Conditional Flow-Based Model for Stochastic Video Generation"},"forum":"rJgUfTEYvH","id":"rJgUfTEYvH"},{"content":{"TLDR":"We study the problem of continuous control agents in deep RL with adversarial attacks and proposed a two-step algorithm based on learned model dynamics. ","abstract":" ","authors":["Tsui-Wei Weng","Krishnamurthy (Dj) Dvijotham","Jonathan Uesato","Kai Xiao","Sven Gowal","Robert Stanforth","Pushmeet Kohli"],"iclr_id":"SylL0krYPS","keywords":["adversarial","adversarial attacks","continuous control","perturbation","reinforcement learning","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Toward Evaluating Robustness of Deep Reinforcement Learning with Continuous Control"},"forum":"SylL0krYPS","id":"SylL0krYPS"},{"content":{"TLDR":"A quantitative refinement of the universal approximation theorem via an algebraic approach.","abstract":" ","authors":["Kai Fong Ernest Chong"],"iclr_id":"rkevSgrtPr","keywords":["universal approximation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A closer look at the approximation capabilities of neural networks"},"forum":"rkevSgrtPr","id":"rkevSgrtPr"},{"content":{"TLDR":"We introduce ReClor, a reading comprehension dataset requiring logical reasoning, and find that current state-of-the-art models struggle with real logical reasoning with poor performance near that of random guess.","abstract":" ","authors":["Weihao Yu","Zihang Jiang","Yanfei Dong","Jiashi Feng"],"iclr_id":"HJgJtT4tvB","keywords":["logical reasoning","nlp","reading comprehension","reasoning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"ReClor: A Reading Comprehension Dataset Requiring Logical Reasoning"},"forum":"HJgJtT4tvB","id":"HJgJtT4tvB"},{"content":{"TLDR":"We introduce DPFRL, a framework for reinforcement learning under partial and complex observations with an importance-weighted particle filter","abstract":" ","authors":["Xiao Ma","Peter Karkus","David Hsu","Wee Sun Lee","Nan Ye"],"iclr_id":"HJl8_eHYvS","keywords":["generative models","navigation","partial observability","reasoning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Discriminative Particle Filter Reinforcement Learning for Complex Partial observations"},"forum":"HJl8_eHYvS","id":"HJl8_eHYvS"},{"content":{"TLDR":"We control the topic and sentiment of text generation (almost) without any training. ","abstract":" ","authors":["Sumanth Dathathri","Andrea Madotto","Janice Lan","Jane Hung","Eric Frank","Piero Molino","Jason Yosinski","Rosanne Liu"],"iclr_id":"H1edEyBKDS","keywords":["fine tuning","generation","generative models","language modeling","nlp","text generation","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Plug and Play Language Models: A Simple Approach to Controlled Text Generation"},"forum":"H1edEyBKDS","id":"H1edEyBKDS"},{"content":{"TLDR":"Addressing task heterogeneity problem in meta-learning by introducing meta-knowledge graph","abstract":" ","authors":["Huaxiu Yao","Xian Wu","Zhiqiang Tao","Yaliang Li","Bolin Ding","Ruirui Li","Zhenhui Li"],"iclr_id":"rklp93EtwH","keywords":["interpretability","meta learning","regression"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Automated Relational Meta-learning"},"forum":"rklp93EtwH","id":"rklp93EtwH"},{"content":{"TLDR":"a feedforward, chain-like motif (1->2->3->...) is proposed as a useful inductive bias for better memory in RNNs.","abstract":" ","authors":["Emin Orhan","Xaq Pitkow"],"iclr_id":"ryx1wRNFvB","keywords":["memory","optimization","rnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Improved memory in recurrent neural networks with sequential non-normal dynamics"},"forum":"ryx1wRNFvB","id":"ryx1wRNFvB"},{"content":{"TLDR":"We extend a pre-trained neural language model by linearly interpolating it with a k-nearest neighbors model, achieving new state-of-the-art results on Wikitext-103 with no additional training.","abstract":" ","authors":["Urvashi Khandelwal","Omer Levy","Dan Jurafsky","Luke Zettlemoyer","Mike Lewis"],"iclr_id":"HklBjCEKvH","keywords":["domain adaptation","generalization","language modeling","nlp"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Generalization through Memorization: Nearest Neighbor Language Models"},"forum":"HklBjCEKvH","id":"HklBjCEKvH"},{"content":{"TLDR":"Hierarchical visual foresight learns to generate visual subgoals that break down long-horizon tasks into subtasks, using only self-supervision.","abstract":" ","authors":["Suraj Nair","Chelsea Finn"],"iclr_id":"H1gzR2VKDH","keywords":["generation","planning","reinforcement learning","scalability","self supervised learning","self supervision","uncertainty","video prediction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Hierarchical Foresight: Self-Supervised Learning of Long-Horizon Tasks via Visual Subgoal Generation"},"forum":"H1gzR2VKDH","id":"H1gzR2VKDH"},{"content":{"TLDR":"We propose a new algorithm that learns constraint-satisfying policies, and provide theoretical analysis and empirical demonstration in the context of reinforcement learning with constraints.","abstract":" ","authors":["Tsung-Yen Yang","Justinian Rosca","Karthik Narasimhan","Peter J. Ramadge"],"iclr_id":"rke3TJrtPS","keywords":["fairness","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Projection-Based Constrained Policy Optimization"},"forum":"rke3TJrtPS","id":"rke3TJrtPS"},{"content":{"TLDR":"We propose a model-based method called \"Search with Amortized Value Estimates\" (SAVE) which leverages both real and planned experience by combining Q-learning with Monte-Carlo Tree Search, achieving strong performance with very small search budgets.","abstract":" ","authors":["Jessica B. Hamrick","Victor Bapst","Alvaro Sanchez-Gonzalez","Tobias Pfaff","Theophane Weber","Lars Buesing","Peter W. Battaglia"],"iclr_id":"SkeAaJrKDS","keywords":["model based rl","planning","reasoning","search"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Combining Q-Learning and Search with Amortized Value Estimates"},"forum":"SkeAaJrKDS","id":"SkeAaJrKDS"},{"content":{"TLDR":"We propose a novel OOD detector that employ blurred images as adversarial examples . Our model achieve significant OOD detection performance in various domains.","abstract":" ","authors":["Sungik Choi","Sae-Young Chung"],"iclr_id":"ByeNra4FDB","keywords":["autoencoder","novelty detection","uncertainty"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Novelty Detection Via Blurring"},"forum":"ByeNra4FDB","id":"ByeNra4FDB"},{"content":{"TLDR":"We pose that generative models' likelihoods are excessively influenced by the input's complexity, and propose a way to compensate it when detecting out-of-distribution inputs","abstract":" ","authors":["Joan Serr\u00e0","David \u00c1lvarez","Vicen\u00e7 G\u00f3mez","Olga Slizovskaia","Jos\u00e9 F. N\u00fa\u00f1ez","Jordi Luque"],"iclr_id":"SyxIWpVYvr","keywords":["generative models","reliability","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Input Complexity and Out-of-distribution Detection with Likelihood-based Generative Models"},"forum":"SyxIWpVYvr","id":"SyxIWpVYvr"},{"content":{"TLDR":"We train a combination of neural networks to predict optimal trajectories for complex physical systems.","abstract":" ","authors":["Philipp Holl","Nils Thuerey","Vladlen Koltun"],"iclr_id":"HyeSin4FPB","keywords":["optimal control","planning"],"recs":[],"session":["Monday: Signals and Systems"],"session_links":[null],"session_times":[null],"title":"Learning to Control PDEs with Differentiable Physics"},"forum":"HyeSin4FPB","id":"HyeSin4FPB"},{"content":{"TLDR":"The data stream model is a fundamental model for processing massive data sets with limited memory and fast processing time. Recently Hsu et al. (2019) incorporated machine learning techniques into the data stream model in order to learn relevant patt...","abstract":" ","authors":["Tanqiu Jiang","Yi Li","Honghao Lin","Yisong Ruan","David P. Woodruff"],"iclr_id":"HyxJ1xBYDH","keywords":["memory"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning-Augmented Data Stream Algorithms"},"forum":"HyxJ1xBYDH","id":"HyxJ1xBYDH"},{"content":{"TLDR":"We propose the first approach that can resist DNN model stealing/extraction attacks","abstract":" ","authors":["Tribhuvanesh Orekondy","Bernt Schiele","Mario Fritz"],"iclr_id":"SyevYxHtDB","keywords":["adversarial","adversarial machine learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Prediction Poisoning: Towards Defenses Against DNN Model Stealing Attacks"},"forum":"SyevYxHtDB","id":"SyevYxHtDB"},{"content":{"TLDR":"Integrating distributed representations with symbolic operations is essential for reading comprehension requiring complex reasoning, such as counting, sorting and arithmetics, but most existing approaches are hard to scale to more domains or more com...","abstract":" ","authors":["Xinyun Chen","Chen Liang","Adams Wei Yu","Denny Zhou","Dawn Song","Quoc V. Le"],"iclr_id":"ryxjnREFwH","keywords":["distributed","question answering","reading comprehension","reasoning"],"recs":[],"session":["Wednesday: Symbols and Discovery"],"session_links":[null],"session_times":[null],"title":"Neural Symbolic Reader: Scalable Integration of Distributed and Symbolic Representations for Reading Comprehension"},"forum":"ryxjnREFwH","id":"ryxjnREFwH"},{"content":{"TLDR":"Learning to align multiple datasets is an important problem with many applications, and it is especially useful when we need to integrate multiple experiments or correct for confounding. Optimal transport (OT) is a principled approach to align  datas...","abstract":" ","authors":["Ruishan Liu","Akshay Balsubramani","James Zou"],"iclr_id":"SJlRUkrFPS","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning transport cost from subset correspondence"},"forum":"SJlRUkrFPS","id":"SJlRUkrFPS"},{"content":{"TLDR":"A sampling-based filter pruning approach for convolutional neural networks exhibiting provable guarantees on the size and performance of the pruned network.","abstract":" ","authors":["Lucas Liebenwein","Cenk Baykal","Harry Lang","Dan Feldman","Daniela Rus"],"iclr_id":"BJxkOlSYDH","keywords":["cnn","compression","importance sampling","pruning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Provable Filter Pruning for Efficient Neural Networks"},"forum":"BJxkOlSYDH","id":"BJxkOlSYDH"},{"content":{"TLDR":"This paper provides novel analysis of adaptive gradient algorithms for solving non-convex non-concave min-max problems as GANs, and explains the reason why adaptive gradient methods outperform its non-adaptive counterparts by empirical studies.","abstract":" ","authors":["Mingrui Liu","Youssef Mroueh","Jerret Ross","Wei Zhang","Xiaodong Cui","Payel Das","Tianbao Yang"],"iclr_id":"SJxIm0VtwH","keywords":["adversarial","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Towards Better Understanding of Adaptive Gradient Algorithms in Generative Adversarial Nets"},"forum":"SJxIm0VtwH","id":"SJxIm0VtwH"},{"content":{"TLDR":"We propose a new notion of margin that has a direct relationship with neural net generalization, and obtain improved generalization bounds for neural nets and robust classification by analyzing this margin.","abstract":" ","authors":["Colin Wei","Tengyu Ma"],"iclr_id":"HJe_yR4Fwr","keywords":["adversarial","deep learning theory","generalization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Improved Sample Complexities for Deep Neural Networks and Robust Classification via an All-Layer Margin"},"forum":"HJe_yR4Fwr","id":"HJe_yR4Fwr"},{"content":{"TLDR":"Four things that improve batch normalization across all batch sizes","abstract":" ","authors":["Cecilia Summers","Michael J. Dinneen"],"iclr_id":"HJx8HANFDH","keywords":["batch normalization","imagenet","reasoning","regularization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Four Things Everyone Should Know to Improve Batch Normalization"},"forum":"HJx8HANFDH","id":"HJx8HANFDH"},{"content":{"TLDR":"A new scalable, group-equivariant model for capsule networks that preserves compositionality under transformations, and is empirically more transformation-robust to older capsule network models.","abstract":" ","authors":["Sai Raam Venkataraman","S. Balasubramanian","R. Raghunatha Sarma"],"iclr_id":"BJgNJgSFPS","keywords":["capsule networks","equivariance","robustness"],"recs":[],"session":["Tuesday: Representation Learning"],"session_links":[null],"session_times":[null],"title":"Building Deep Equivariant Capsule Networks"},"forum":"BJgNJgSFPS","id":"BJgNJgSFPS"},{"content":{"TLDR":"We relate the asymptotic behavior of graph neural networks to the graph spectra of underlying graphs and gives principled guidelines for normalizing weights.","abstract":" ","authors":["Kenta Oono","Taiji Suzuki"],"iclr_id":"S1ldO2EFPr","keywords":["expressive power","graph networks"],"recs":[],"session":["Wednesday: Theory"],"session_links":[null],"session_times":[null],"title":"Graph Neural Networks Exponentially Lose Expressive Power for Node Classification"},"forum":"S1ldO2EFPr","id":"S1ldO2EFPr"},{"content":{"TLDR":"Black-box attack methods aim to infer suitable attack patterns to targeted DNN models by only using output feedback of the models and the corresponding input queries. However, due to lack of prior and inefficiency in leveraging the query and feedback...","abstract":" ","authors":["Jiawei Du","Hu Zhang","Joey Tianyi Zhou","Yi Yang","Jiashi Feng"],"iclr_id":"Skxd6gSYDS","keywords":["adversarial","imagenet","meta learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Query-efficient Meta Attack to Deep Neural Networks"},"forum":"Skxd6gSYDS","id":"Skxd6gSYDS"},{"content":{"TLDR":"The first rigor diagnose of large-scale adversarial training on ImageNet","abstract":" ","authors":["Cihang Xie","Alan Yuille"],"iclr_id":"HyxJhCEFDS","keywords":["adversarial","adversarial attacks","adversarial defense","adversarial machine learning","batch normalization","capacity","imagenet","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Intriguing Properties of Adversarial Training at Scale"},"forum":"HyxJhCEFDS","id":"HyxJhCEFDS"},{"content":{"TLDR":"We introduce PAU, a new learnable activation function for neural networks. They free the network designers from the activation selection process and increase the test prediction accuracy.","abstract":" ","authors":["Alejandro Molina","Patrick Schramowski","Kristian Kersting"],"iclr_id":"BJlBSkHtDS","keywords":["robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Pad\u00e9 Activation Units: End-to-end Learning of Flexible Activation Functions in Deep Networks"},"forum":"BJlBSkHtDS","id":"BJlBSkHtDS"},{"content":{"TLDR":" We propose to learn an input-dependent subgraph, dynamically and selectively expanded, to explicitly model a sequential reasoning process.","abstract":" ","authors":["Xiaoran Xu","Wei Feng","Yunsheng Jiang","Xiaohui Xie","Zhiqing Sun","Zhi-Hong Deng"],"iclr_id":"rkeuAhVKvB","keywords":["attention","graph networks","knowledge base completion","reasoning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Dynamically Pruned Message Passing Networks for Large-scale Knowledge Graph Reasoning"},"forum":"rkeuAhVKvB","id":"rkeuAhVKvB"},{"content":{"TLDR":"We develop a theoretical framework to characterize what a neural network can learn to reason about.","abstract":" ","authors":["Keyulu Xu","Jingling Li","Mozhi Zhang","Simon S. Du","Ken-ichi Kawarabayashi","Stefanie Jegelka"],"iclr_id":"rJxbJeHFPS","keywords":["deep learning theory","expressive power","graph networks","question answering","reasoning"],"recs":[],"session":["Wednesday: Theory"],"session_links":[null],"session_times":[null],"title":"What Can Neural Networks Reason About?"},"forum":"rJxbJeHFPS","id":"rJxbJeHFPS"},{"content":{"TLDR":"We propose a structured object-aware video prediction model, which explicitly reasons about objects and demonstrate that it provides high-quality long term video predictions for planning.","abstract":" ","authors":["Jannik Kossen","Karl Stelzner","Marcel Hussing","Claas Voelcker","Kristian Kersting"],"iclr_id":"B1e-kxSKDH","keywords":["graph networks","model based reinforcement learning","planning","reinforcement learning","relational learning","search","self supervised learning","unsupervised","video prediction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Structured Object-Aware Physics Prediction for Video Modeling and Planning"},"forum":"B1e-kxSKDH","id":"B1e-kxSKDH"},{"content":{"TLDR":"We propose an event-based policy gradient  to train the leader and an action abstraction policy gradient to train the followers in leader-follower Markov game.","abstract":" ","authors":["Zhenyu Shi","Runsheng Yu","Xinrun Wang","Rundong Wang","Youzhi Zhang","Hanjiang Lai","Bo An"],"iclr_id":"ryeG924twB","keywords":["attention","multi agent reinforcement learning","navigation","policy gradient","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning Expensive Coordination: An Event-Based Deep RL Approach"},"forum":"ryeG924twB","id":"ryeG924twB"},{"content":{"TLDR":"Is it possible to co-design model accuracy, robustness and efficiency to achieve their triple wins? Yes!","abstract":" ","authors":["Ting-Kuei Hu","Tianlong Chen","Haotao Wang","Zhangyang Wang"],"iclr_id":"rJgzzJHtDB","keywords":["adversarial","adversarial attacks","capacity","efficient inference","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Triple Wins: Boosting Accuracy, Robustness and Efficiency Together by Enabling Input-Adaptive Inference"},"forum":"rJgzzJHtDB","id":"rJgzzJHtDB"},{"content":{"TLDR":"We augment the Q-value estimates with a count-based bonus that ensures optimism during action selection and bootstrapping, even if the Q-value estimates are pessimistic.","abstract":" ","authors":["Tabish Rashid","Bei Peng","Wendelin Boehmer","Shimon Whiteson"],"iclr_id":"r1xGP6VYwH","keywords":["intrinsic motivation","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Optimistic Exploration even with a Pessimistic Initialisation"},"forum":"r1xGP6VYwH","id":"r1xGP6VYwH"},{"content":{"TLDR":"Derived population risk of two-layer neural networks in high dimensions and examined presence / absence of  \"double descent\".","abstract":" ","authors":["Jimmy Ba","Murat Erdogdu","Taiji Suzuki","Denny Wu","Tianzong Zhang"],"iclr_id":"H1gBsgBYwH","keywords":["generalization","inductive bias","regression"],"recs":[],"session":["Monday: Optimisation Theory"],"session_links":[null],"session_times":[null],"title":"Generalization of Two-layer Neural Networks: An Asymptotic Viewpoint"},"forum":"H1gBsgBYwH","id":"H1gBsgBYwH"},{"content":{"TLDR":"We use deep RL to learn a policy that directs the search of a genetic algorithm to better optimize the execution cost of computation graphs, and show improved results on real-world TensorFlow graphs.","abstract":" ","authors":["Aditya Paliwal","Felix Gimeno","Vinod Nair","Yujia Li","Miles Lubin","Pushmeet Kohli","Oriol Vinyals"],"iclr_id":"rkxDoJBYPB","keywords":["combinatorial optimization","learning for systems","memory","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Reinforced Genetic Algorithm Learning for Optimizing Computation Graphs"},"forum":"rkxDoJBYPB","id":"rkxDoJBYPB"},{"content":{"TLDR":"We proposed a progressive learning method to improve learning and disentangling latent representations at different levels of abstraction.","abstract":" ","authors":["Zhiyuan Li","Jaideep Vitthal Murkute","Prashnna Kumar Gyawali","Linwei Wang"],"iclr_id":"SJxpsxrYPS","keywords":["capacity","disentanglement","generation","generative models","mutual information","representation learning","variational inference"],"recs":[],"session":["Tuesday: Representation Learning"],"session_links":[null],"session_times":[null],"title":"PROGRESSIVE LEARNING AND DISENTANGLEMENT OF HIERARCHICAL REPRESENTATIONS"},"forum":"SJxpsxrYPS","id":"SJxpsxrYPS"},{"content":{"TLDR":"Intrinsically motivated reinforcement learning aims to address the exploration challenge for sparse-reward tasks. However, the study of exploration methods in transition-dependent multi-agent settings is largely absent from the literature. We aim to ...","abstract":" ","authors":["Tonghan Wang","Jianhao Wang","Yi Wu","Chongjie Zhang"],"iclr_id":"BJgy96EYvr","keywords":["multi agent reinforcement learning","mutual information","policy gradient","reinforcement learning"],"recs":[],"session":["Monday: Multiagent Systems"],"session_links":[null],"session_times":[null],"title":"Influence-Based Multi-Agent Exploration"},"forum":"BJgy96EYvr","id":"BJgy96EYvr"},{"content":{"TLDR":"We develop a new self-attention based graph neural network called Hyper-SAGNN applicable to homogeneous and heterogeneous hypergraphs with variable hyperedge sizes that can fulfill tasks like node classification and hyperedge prediction. ","abstract":" ","authors":["Ruochi Zhang","Yuesong Zou","Jian Ma"],"iclr_id":"ryeHuJBtPH","keywords":["attention","graph networks","representation learning","self attention"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Hyper-SAGNN: a self-attention based graph neural network for hypergraphs"},"forum":"ryeHuJBtPH","id":"ryeHuJBtPH"},{"content":{"TLDR":"We study how an unsupervised exploration and feature learning approach addresses efficiently a new problem: automatic discovery of diverse self-organized patterns in high-dim complex systems such as the game of life.","abstract":" ","authors":["Chris Reinke","Mayalen Etcheverry","Pierre-Yves Oudeyer"],"iclr_id":"rkg6sJHYDr","keywords":["robotics","unsupervised","exploration","object discovery","autoencoder","sample efficiency","intrinsic motivation"],"recs":[],"session":["Wednesday: Symbols and Discovery"],"session_links":[null],"session_times":[null],"title":"Intrinsically Motivated Discovery of Diverse Patterns in Self-Organizing Systems"},"forum":"rkg6sJHYDr","id":"rkg6sJHYDr"},{"content":{"TLDR":"Generative adversarial training a new generative modeling technique and its applications to adversarial example detection and robust classification.","abstract":" ","authors":["Xuwang Yin","Soheil Kolouri","Gustavo K Rohde"],"iclr_id":"SJeQEp4YDH","keywords":["adversarial","generative models","optimization","security"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"GAT: Generative Adversarial Training for Adversarial Example Detection and Classification"},"forum":"SJeQEp4YDH","id":"SJeQEp4YDH"},{"content":{"TLDR":"We make two theoretical contributions to disentanglement learning by (a) defining precise semantics of disentangled representations, and (b) establishing robust metrics for evaluation. First, we characterize the concept \u201cdisentangled representations\u201d...","abstract":" ","authors":["Kien Do","Truyen Tran"],"iclr_id":"HJgK0h4Ywr","keywords":["disentanglement","interpretability","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Theory and Evaluation Metrics for Learning Disentangled Representations"},"forum":"HJgK0h4Ywr","id":"HJgK0h4Ywr"},{"content":{"TLDR":"Make the transformer streamable with monotonic attention.","abstract":" ","authors":["Xutai Ma","Juan Miguel Pino","James Cross","Liezl Puzon","Jiatao Gu"],"iclr_id":"Hyg96gBKPS","keywords":["attention","machine translation","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Monotonic Multihead Attention"},"forum":"Hyg96gBKPS","id":"Hyg96gBKPS"},{"content":{"TLDR":"We present a diagnostic dataset for systematic study of temporal and casual reasoning in videos. ","abstract":" ","authors":["Kexin Yi","Chuang Gan","Yunzhu Li","Pushmeet Kohli","Jiajun Wu","Antonio Torralba","Joshua B. Tenenbaum"],"iclr_id":"HkxYzANYDB","keywords":["nlp","reasoning"],"recs":[],"session":["Wednesday: Actions and Counterfactuals"],"session_links":[null],"session_times":[null],"title":"CLEVRER: Collision Events for Video Representation and Reasoning"},"forum":"HkxYzANYDB","id":"HkxYzANYDB"},{"content":{"TLDR":"A Theoretical Study of Multi-Task Learning with Practical Implications for Improving Multi-Task Training and Transfer Learning","abstract":" ","authors":["Sen Wu","Hongyang Zhang","Christopher R\u00e9"],"iclr_id":"SylzhkBtDB","keywords":["multi task learning","robustness","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Understanding and Improving Information Transfer in Multi-Task Learning"},"forum":"SylzhkBtDB","id":"SylzhkBtDB"},{"content":{"TLDR":"This paper proposes a new formulation and a new communication protocol for networked multi-agent control problems","abstract":" ","authors":["Tianshu Chu","Sandeep Chinchali","Sachin Katti"],"iclr_id":"Syx7A3NFvH","keywords":["multi agent reinforcement learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Multi-agent Reinforcement Learning for Networked System Control"},"forum":"Syx7A3NFvH","id":"Syx7A3NFvH"},{"content":{"TLDR":"Overparameterized neural networks can be distributionally robust, but only when you account for generalization. ","abstract":" ","authors":["Shiori Sagawa","Pang Wei Koh","Tatsunori B. Hashimoto","Percy Liang"],"iclr_id":"ryxGuJrFvS","keywords":["generalization","nlp","optimization","regularization","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Distributionally Robust Neural Networks"},"forum":"ryxGuJrFvS","id":"ryxGuJrFvS"},{"content":{"TLDR":"Multivariate spatial point process models can describe heterotopic data over space. However, highly multivariate intensities are computationally challenging due to the curse of dimensionality. To bridge this gap, we introduce a declustering based hid...","abstract":" ","authors":["Baichuan Yuan","Xiaowei Wang","Jianxin Ma","Chang Zhou","Andrea L. Bertozzi","Hongxia Yang"],"iclr_id":"B1lj20NFDS","keywords":["autoencoder","clustering","collaborative filtering","generalization","recommender systems","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Variational Autoencoders for Highly Multivariate Spatial Point Processes Intensities"},"forum":"B1lj20NFDS","id":"B1lj20NFDS"},{"content":{"TLDR":"We introduce techniques to train a single once-for-all network that fits many hardware platforms.","abstract":" ","authors":["Han Cai","Chuang Gan","Tianzhe Wang","Zhekai Zhang","Song Han"],"iclr_id":"HylxE1HKwS","keywords":["automl","imagenet","neural architecture search"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Once for All: Train One Network and Specialize it for Efficient Deployment"},"forum":"HylxE1HKwS","id":"HylxE1HKwS"},{"content":{"TLDR":"We propose a query-efficient black-box attack which uses Bayesian optimisation in combination with Bayesian model selection to optimise over the adversarial perturbation and the optimal degree of search space dimension reduction. ","abstract":" ","authors":["Binxin Ru","Adam Cobb","Arno Blaas","Yarin Gal"],"iclr_id":"Hkem-lrtvH","keywords":["adversarial","adversarial attacks","black box adversarial attack","gaussian process","perturbation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"BayesOpt Adversarial Attack"},"forum":"Hkem-lrtvH","id":"Hkem-lrtvH"},{"content":{"TLDR":"Latent Optimization for Representation Disentanglement","abstract":" ","authors":["Aviv Gabbay","Yedid Hoshen"],"iclr_id":"Hyl9xxHYPr","keywords":["adversarial","clustering","disentanglement","optimization","regularization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Demystifying Inter-Class Disentanglement"},"forum":"Hyl9xxHYPr","id":"Hyl9xxHYPr"},{"content":{"TLDR":"This paper is concerned with solving combinatorial optimization problems, in particular, the capacitated vehicle routing problems (CVRP). Classical Operations Research (OR) algorithms such as LKH3 (Helsgaun, 2017) are extremely inefficient (e.g., 13 ...","abstract":" ","authors":["Hao Lu","Xingwen Zhang","Shuang Yang"],"iclr_id":"BJe1334YDH","keywords":["optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Learning-based Iterative Method for Solving Vehicle Routing Problems"},"forum":"BJe1334YDH","id":"BJe1334YDH"},{"content":{"TLDR":"We present a neural memory-based architecture for incremental domain adaptation, and provide theoretical and empirical results.","abstract":" ","authors":["Nabiha Asghar","Lili Mou","Kira A. Selby","Kevin D. Pantasdo","Pascal Poupart","Xin Jiang"],"iclr_id":"BkepbpNFwr","keywords":["attention","capacity","domain adaptation","fine tuning","memory","nlp"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Progressive Memory Banks for Incremental Domain Adaptation"},"forum":"BkepbpNFwr","id":"BkepbpNFwr"},{"content":{"TLDR":"We learn an efficient lossy image codec that can be optimized to facilitate reliable photo manipulation detection at fractional cost in payload/quality and even at low bitrates.","abstract":" ","authors":["Pawel Korus","Nasir Memon"],"iclr_id":"HyxG3p4twS","keywords":["compression","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Quantifying the Cost of Reliable Photo Authentication via High-Performance Learned Lossy Representations"},"forum":"HyxG3p4twS","id":"HyxG3p4twS"},{"content":{"TLDR":"Generative Adversarial Imitation Learning (GAIL) is a powerful and practical approach for learning sequential decision-making policies. Different from Reinforcement Learning (RL), GAIL takes advantage of demonstration data by experts (e.g., human), a...","abstract":" ","authors":["Minshuo Chen","Yizhou Wang","Tianyi Liu","Zhuoran Yang","Xingguo Li","Zhaoran Wang","Tuo Zhao"],"iclr_id":"BJl-5pNKDB","keywords":["adversarial","generalization","imitation learning","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On Computation and Generalization of Generative Adversarial Imitation Learning"},"forum":"BJl-5pNKDB","id":"BJl-5pNKDB"},{"content":{"TLDR":" Recent breakthroughs of pretrained language models have shown the effectiveness of self-supervised learning for a wide range of natural language processing (NLP) tasks. In addition to standard syntactic and semantic NLP tasks, pretrained models achi...","abstract":" ","authors":["Wenhan Xiong","Jingfei Du","William Yang Wang","Veselin Stoyanov"],"iclr_id":"BJlzm64tDH","keywords":["language modeling","nlp","question answering","self supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Pretrained Encyclopedia: Weakly Supervised Knowledge-Pretrained Language Model"},"forum":"BJlzm64tDH","id":"BJlzm64tDH"},{"content":{"TLDR":"We propose the first non-autoregressive neural model for Dialogue State Tracking (DST), achieving the SOTA accuracy (49.04%) on MultiWOZ2.1 benchmark, and reducing inference latency by an order of magnitude.","abstract":" ","authors":["Hung Le","Richard Socher","Steven C.H. Hoi"],"iclr_id":"H1e_cC4twS","keywords":["generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Non-Autoregressive Dialog State Tracking"},"forum":"H1e_cC4twS","id":"H1e_cC4twS"},{"content":{"TLDR":"We introduce the idea of adversarial learning into automatic data augmentation to improve the generalization  of a targe network.","abstract":" ","authors":["Xinyu Zhang","Qiang Wang","Jian Zhang","Zhao Zhong"],"iclr_id":"ByxdUySKvS","keywords":["adversarial","generalization","imagenet","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Adversarial AutoAugment"},"forum":"ByxdUySKvS","id":"ByxdUySKvS"},{"content":{"TLDR":"A meta-learning method for fast adaptation of neural architectures.","abstract":" ","authors":["Dongze Lian","Yin Zheng","Yintao Xu","Yanxiong Lu","Leyu Lin","Peilin Zhao","Junzhou Huang","Shenghua Gao"],"iclr_id":"r1eowANFvr","keywords":["fewshot learning","meta learning","neural architecture search","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Towards Fast Adaptation of Neural Architectures with Meta Learning"},"forum":"r1eowANFvr","id":"r1eowANFvr"},{"content":{"TLDR":"We proposed a double neural framework to solve large-scale imperfect information game. ","abstract":" ","authors":["Hui Li","Kailiang Hu","Shaohua Zhang","Yuan Qi","Le Song"],"iclr_id":"ByedzkrKvH","keywords":["memory","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Double Neural Counterfactual Regret Minimization"},"forum":"ByedzkrKvH","id":"ByedzkrKvH"},{"content":{"TLDR":"Video retargeting typically requires large amount of target data to be effective, which may not always be available; we propose a metalearning approach that improves over popular baselines while producing temporally coherent frames.","abstract":" ","authors":["Jessica Lee","Deva Ramanan","Rohit Girdhar"],"iclr_id":"SJx1URNKwH","keywords":["adversarial","fewshot learning","gan","generative models","meta learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"MetaPix: Few-Shot Video Retargeting"},"forum":"SJx1URNKwH","id":"SJx1URNKwH"},{"content":{"TLDR":"An implementation of the backpropagation algorithm using spiking neurons for forward and backward propagation.","abstract":" ","authors":["Johannes C. Thiele","Olivier Bichler","Antoine Dupret"],"iclr_id":"rkxs0yHFPH","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"SpikeGrad: An ANN-equivalent Computation Model for Implementing Backpropagation with Spikes"},"forum":"rkxs0yHFPH","id":"rkxs0yHFPH"},{"content":{"TLDR":"SEED RL, a scalable and efficient deep reinforcement learning agent with accelerated central inference. State of the art results, reduces cost and can process millions of frames per second. ","abstract":" ","authors":["Lasse Espeholt","Rapha\u00ebl Marinier","Piotr Stanczyk","Ke Wang","Marcin Michalski\u200e"],"iclr_id":"rkgvXlrKwH","keywords":["distributed","policy gradient","reinforcement learning","scalability"],"recs":[],"session":["Tuesday: RL and Estimation"],"session_links":[null],"session_times":[null],"title":"SEED RL: Scalable and Efficient Deep-RL with Accelerated Central Inference"},"forum":"rkgvXlrKwH","id":"rkgvXlrKwH"},{"content":{"TLDR":"We develop a new storage assignment framework with a novel neural network that enables large efficiency gains in the warehouse.","abstract":" ","authors":["Michael Lingzhi Li","Elliott Wolf","Daniel Wintz"],"iclr_id":"Hkx7xRVYDr","keywords":["nlp","uncertainty"],"recs":[],"session":["Wednesday: Sequence Representations"],"session_links":[null],"session_times":[null],"title":"Duration-of-Stay Storage Assignment under Uncertainty"},"forum":"Hkx7xRVYDr","id":"Hkx7xRVYDr"},{"content":{"TLDR":"We propose a generative latent variable model for unsupervised scene decomposition that provides factorized object representation per foreground object while also decomposing background segments of complex morphology.","abstract":" ","authors":["Zhixuan Lin","Yi-Fu Wu","Skand Vishwanath Peri","Weihao Sun","Gautam Singh","Fei Deng","Jindong Jiang","Sungjin Ahn"],"iclr_id":"rkl03ySYDH","keywords":["attention","generative models","representation learning","scalability","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"SPACE: Unsupervised Object-Oriented Scene Representation via Spatial Attention and Decomposition"},"forum":"rkl03ySYDH","id":"rkl03ySYDH"},{"content":{"TLDR":"Contextual illusions are a feature, not a bug, of neural routines optimized for contour detection.","abstract":" ","authors":["Drew Linsley","Junkyung Kim","Alekh Ashok","Thomas Serre"],"iclr_id":"H1gB4RVKvB","keywords":["computer vision","neural circuits","recurrent feedback","sample efficiency","visual cortex"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Recurrent neural circuits for contour detection"},"forum":"H1gB4RVKvB","id":"H1gB4RVKvB"},{"content":{"TLDR":"We study the roots of algorithmic progress in deep policy gradient algorithms through a case study on two popular algorithms, Proximal Policy Optimization and Trust Region Policy Optimization. We investigate the consequences of \"code-level optimizati...","abstract":" ","authors":["Logan Engstrom","Andrew Ilyas","Shibani Santurkar","Dimitris Tsipras","Firdaus Janoos","Larry Rudolph","Aleksander Madry"],"iclr_id":"r1etN1rtPB","keywords":["optimization","policy gradient","reinforcement learning"],"recs":[],"session":["Monday: Reliable RL"],"session_links":[null],"session_times":[null],"title":"Implementation Matters in Deep RL: A Case Study on PPO and TRPO"},"forum":"r1etN1rtPB","id":"r1etN1rtPB"},{"content":{"TLDR":"We propose BERTScore, an automatic evaluation metric for text generation, which correlates better with human judgments and provides stronger model selection performance than existing metrics.","abstract":" ","authors":["Tianyi Zhang","Varsha Kishore","Felix Wu","Kilian Q. Weinberger","Yoav Artzi"],"iclr_id":"SkeHuCVFDr","keywords":["adversarial","generation","machine translation","text generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"BERTScore: Evaluating Text Generation with BERT"},"forum":"SkeHuCVFDr","id":"SkeHuCVFDr"},{"content":{"TLDR":"We address the task of autonomous exploration and navigation using spatial affordance maps that can be learned in a self-supervised manner, these outperform classic geometric baselines while being more sample efficient than contemporary RL algorithms","abstract":" ","authors":["William Qi","Ravi Teja Mullapudi","Saurabh Gupta","Deva Ramanan"],"iclr_id":"BJgMFxrYPB","keywords":["navigation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning to Move with Affordance Maps"},"forum":"BJgMFxrYPB","id":"BJgMFxrYPB"},{"content":{"TLDR":"Exponential lower bounds for value-based and policy-based reinforcement learning with function approximation.","abstract":" ","authors":["Simon S. Du","Sham M. Kakade","Ruosong Wang","Lin F. Yang"],"iclr_id":"r1genAVKPB","keywords":["imitation learning","reinforcement learning"],"recs":[],"session":["Tuesday: RL and Estimation"],"session_links":[null],"session_times":[null],"title":"Is a Good Representation Sufficient for Sample Efficient Reinforcement Learning?"},"forum":"r1genAVKPB","id":"r1genAVKPB"},{"content":{"TLDR":"We propose a method called network deconvolution that resembles animal vision system to train convolution networks better.","abstract":" ","authors":["Chengxi Ye","Matthew Evanusa","Hua He","Anton Mitrokhin","Tom Goldstein","James A. Yorke","Cornelia Fermuller","Yiannis Aloimonos"],"iclr_id":"rkeu30EtvS","keywords":["batch normalization","cnn","imagenet"],"recs":[],"session":["Thursday: Network Architectures"],"session_links":[null],"session_times":[null],"title":"Network Deconvolution"},"forum":"rkeu30EtvS","id":"rkeu30EtvS"},{"content":{"TLDR":"Learning an implicit master policy, as a master policy in HRL can fail to generalize.","abstract":" ","authors":["Anirudh Goyal","Shagun Sodhani","Jonathan Binas","Xue Bin Peng","Sergey Levine","Yoshua Bengio"],"iclr_id":"ryxgJTEYDr","keywords":["ensembles","generalization","hierarchical reinforcement learning","information bottleneck","reinforcement learning","variational information bottleneck"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Reinforcement Learning with Competitive  Ensembles of Information-Constrained Primitives"},"forum":"ryxgJTEYDr","id":"ryxgJTEYDr"},{"content":{"TLDR":"We study the problem of learning and optimizing through physical simulations via differentiable programming, using our proposed DiffSim programming language and compiler.","abstract":" ","authors":["Yuanming Hu","Luke Anderson","Tzu-Mao Li","Qi Sun","Nathan Carr","Jonathan Ragan-Kelley","Fredo Durand"],"iclr_id":"B1eB5xSFvr","keywords":["nlp","optimal control","optimization","robotics"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"DiffTaichi: Differentiable Programming for Physical Simulation"},"forum":"B1eB5xSFvr","id":"B1eB5xSFvr"},{"content":{"TLDR":"We show state-of-the-art word representation learning methods maximize an objective function that is a lower bound on the mutual information between different parts of a word sequence (i.e., a sentence). Our formulation provides an alternative perspe...","abstract":" ","authors":["Lingpeng Kong","Cyprien de Masson d'Autume","Lei Yu","Wang Ling","Zihang Dai","Dani Yogatama"],"iclr_id":"Syx79eBKwr","keywords":["computer vision","mutual information","nlp","representation learning","word embedding"],"recs":[],"session":["Wednesday: Sequence Representations"],"session_links":[null],"session_times":[null],"title":"A Mutual Information Maximization Perspective of Language Representation Learning"},"forum":"Syx79eBKwr","id":"Syx79eBKwr"},{"content":{"TLDR":"Modeling complex multi-agent interactions under multi-agent imitation learning framework with explicit modeling of correlated policies by approximating opponents\u2019 policies. ","abstract":" ","authors":["Minghuan Liu","Ming Zhou","Weinan Zhang","Yuzheng Zhuang","Jun Wang","Wulong Liu","Yong Yu"],"iclr_id":"B1gZV1HYvS","keywords":["adversarial","imitation learning","multi agent reinforcement learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Multi-Agent Interactions Modeling with Correlated Policies"},"forum":"B1gZV1HYvS","id":"B1gZV1HYvS"},{"content":{"TLDR":"Consider an imitation learning problem that the imitator and the expert have different dynamics models. Most of existing imitation learning methods fail because they focus on the imitation of actions. We propose a novel state alignment-based imitatio...","abstract":" ","authors":["Fangchen Liu","Zhan Ling","Tongzhou Mu","Hao Su"],"iclr_id":"rylrdxHFDr","keywords":["imitation learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"State Alignment-based Imitation Learning"},"forum":"rylrdxHFDr","id":"rylrdxHFDr"},{"content":{"TLDR":"We propose a novel combination of adversarial training and provable defenses which produces a model with state-of-the-art accuracy and certified robustness on CIFAR-10. ","abstract":" ","authors":["Mislav Balunovic","Martin Vechev"],"iclr_id":"SJxSDxrKDr","keywords":["adversarial","perturbation","robustness","verification","adversarial"],"recs":[],"session":["Monday: Robustness and Verification"],"session_links":[null],"session_times":[null],"title":"Adversarial Training and Provable Defenses: Bridging the Gap"},"forum":"SJxSDxrKDr","id":"SJxSDxrKDr"},{"content":{"TLDR":"Benchmark and method to measure compositional generalization by maximizing divergence of compound frequency at small divergence of atom frequency.","abstract":" ","authors":["Daniel Keysers","Nathanael Sch\u00e4rli","Nathan Scales","Hylke Buisman","Daniel Furrer","Sergii Kashubin","Nikola Momchev","Danila Sinopalnikov","Lukasz Stafiniak","Tibor Tihon","Dmitry Tsarkov","Xiao Wang","Marc van Zee","Olivier Bousquet"],"iclr_id":"SygcCnNKwr","keywords":["compositionality","generalization","nlp","question answering"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Measuring Compositional Generalization: A Comprehensive Method on Realistic Data"},"forum":"SygcCnNKwr","id":"SygcCnNKwr"},{"content":{"TLDR":"This work presents a method of generating and using ensembles effectively to identify noisy examples in the presence of annotation noise. ","abstract":" ","authors":["Jisoo Lee","Sae-Young Chung"],"iclr_id":"ryxOUTVYDH","keywords":["overfitting","perturbation","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Robust training with ensemble consensus"},"forum":"ryxOUTVYDH","id":"ryxOUTVYDH"},{"content":{"TLDR":"A regularization-based approach for continual learning using Bayesian neural networks to predict parameters' importance","abstract":" ","authors":["Sayna Ebrahimi","Mohamed Elhoseiny","Trevor Darrell","Marcus Rohrbach"],"iclr_id":"HklUCCVKDB","keywords":["capacity","catastrophic forgetting","continual learning","learning rate","pruning","regularization","uncertainty"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Uncertainty-guided Continual Learning with Bayesian Neural Networks"},"forum":"HklUCCVKDB","id":"HklUCCVKDB"},{"content":{"TLDR":"We study the adversarial machine learning attacks against the Multiple Object Tracking mechanisms for the first time. ","abstract":" ","authors":["Yunhan Jia","Yantao Lu","Junjie Shen","Qi Alfred Chen","Hao Chen","Zhenyu Zhong","Tao Wei"],"iclr_id":"rJl31TNYPr","keywords":["adversarial","adversarial machine learning","autonomous driving","object detection","security"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Fooling Detection Alone is Not Enough: Adversarial Attack against Multiple Object Tracking"},"forum":"rJl31TNYPr","id":"rJl31TNYPr"},{"content":{"TLDR":"Incorporating hierarchical structures like constituency trees has been shown to be effective for various natural language processing (NLP) tasks. However, it is evident that state-of-the-art (SOTA) sequence-based models like the Transformer struggle ...","abstract":" ","authors":["Xuan-Phi Nguyen","Shafiq Joty","Steven Hoi","Richard Socher"],"iclr_id":"HJxK5pEYvr","keywords":["attention","machine translation","nlp","self attention","sentiment analysis","text classification","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Tree-Structured Attention with Hierarchical Accumulation"},"forum":"HJxK5pEYvr","id":"HJxK5pEYvr"},{"content":{"TLDR":"We propose an unsupervised skill discovery which enables model-based planning for hierarchical reinforcement learning.","abstract":" ","authors":["Archit Sharma","Shixiang Gu","Sergey Levine","Vikash Kumar","Karol Hausman"],"iclr_id":"HJgLZR4KvH","keywords":["hierarchical reinforcement learning","model based reinforcement learning","planning","reinforcement learning","skill discovery","unsupervised"],"recs":[],"session":["Thursday: Continual Learning and Few Shot Learning"],"session_links":[null],"session_times":[null],"title":"Dynamics-Aware Unsupervised Skill Discovery"},"forum":"HJgLZR4KvH","id":"HJgLZR4KvH"},{"content":{"TLDR":"Popular algorithms that cast \"RL as Inference\" ignore the role of uncertainty and exploration. We highlight the importance of these issues and present a coherent framework for RL and inference that handles them gracefully.","abstract":" ","authors":["Brendan O'Donoghue","Ian Osband","Catalin Ionescu"],"iclr_id":"S1xitgHtvS","keywords":["bayesian inference","reinforcement learning"],"recs":[],"session":["Monday: Reliable RL"],"session_links":[null],"session_times":[null],"title":"Making Sense of Reinforcement Learning and Probabilistic Inference"},"forum":"S1xitgHtvS","id":"S1xitgHtvS"},{"content":{"TLDR":"Variational approaches based on neural networks are showing promise for estimating mutual information (MI) between high dimensional variables. However, they can be difficult to use in practice due to poorly understood bias/variance tradeoffs. We theo...","abstract":" ","authors":["Jiaming Song","Stefano Ermon"],"iclr_id":"B1x62TNtDS","keywords":["mutual information","variance reduction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Understanding the Limitations of Variational Mutual Information Estimators"},"forum":"B1x62TNtDS","id":"B1x62TNtDS"},{"content":{"TLDR":"A novel approach using mode connectivity in loss landscapes to mitigate adversarial effects, repair tampered models, and evaluate adversarial robustness","abstract":" ","authors":["Pu Zhao","Pin-Yu Chen","Payel Das","Karthikeyan Natesan Ramamurthy","Xue Lin"],"iclr_id":"SJgwzCEKwH","keywords":["adversarial","adversarial attacks","loss landscape","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Bridging Mode Connectivity in Loss Landscapes and Adversarial Robustness"},"forum":"SJgwzCEKwH","id":"SJgwzCEKwH"},{"content":{"TLDR":"Reward agents for taking actions that lead to changes in the environment state.","abstract":" ","authors":["Roberta Raileanu","Tim Rockt\u00e4schel"],"iclr_id":"rkg-TJBFPB","keywords":["curiosity","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"RIDE: Rewarding Impact-Driven Exploration for Procedurally-Generated Environments"},"forum":"rkg-TJBFPB","id":"rkg-TJBFPB"},{"content":{"TLDR":"We propose a formulation of intrinsic motivation that is suitable as an exploration bias in synergistic multi-agent tasks, by encouraging agents to affect the world in ways that would not be achieved if they were acting individually.","abstract":" ","authors":["Rohan Chitnis","Shubham Tulsiani","Saurabh Gupta","Abhinav Gupta"],"iclr_id":"SJleNCNtDH","keywords":["intrinsic motivation","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Intrinsic Motivation for Encouraging Synergistic Behavior"},"forum":"SJleNCNtDH","id":"SJleNCNtDH"},{"content":{"TLDR":"We study non-collaborative dialogs, where two agents have a conflict of interest but must strategically communicate to reach an agreement (e.g., negotiation). This setting poses new challenges for modeling dialog history because the dialog's outcome ...","abstract":" ","authors":["Yiheng Zhou","Yulia Tsvetkov","Alan W Black","Zhou Yu"],"iclr_id":"ryxQuANKPB","keywords":["generation","interpretability","planning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Augmenting Non-Collaborative Dialog Systems with Explicit Semantic and Strategic Dialog History"},"forum":"ryxQuANKPB","id":"ryxQuANKPB"},{"content":{"TLDR":"We give a theoretical analysis of the Information Bottleneck objective to understand and predict observed phase transitions in the prediction vs. compression tradeoff.","abstract":" ","authors":["Tailin Wu","Ian Fischer"],"iclr_id":"HJloElBYvB","keywords":["compression","information bottleneck","information theory","loss landscape","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Phase Transitions for the Information Bottleneck in Representation Learning"},"forum":"HJloElBYvB","id":"HJloElBYvB"},{"content":{"TLDR":"We present an efficient and adaptive framework for comparing image classifiers to maximize the discrepancies between the classifiers, in place of comparing on fixed test sets.","abstract":" ","authors":["Haotao Wang","Tianlong Chen","Zhangyang Wang","Kede Ma"],"iclr_id":"rJehNT4YPr","keywords":["distributed","imagenet"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"I Am Going MAD: Maximum Discrepancy Competition for Comparing Classifiers Adaptively"},"forum":"rJehNT4YPr","id":"rJehNT4YPr"},{"content":{"TLDR":"Many tasks in modern machine learning can be formulated as finding equilibria in sequential games. In particular, two-player zero-sum sequential games, also known as minimax optimization, have received growing interest. It is tempting to apply gradie...","abstract":" ","authors":["Yuanhao Wang","Guodong Zhang","Jimmy Ba"],"iclr_id":"Hkx7_1rKwS","keywords":["adversarial","gan","gradient descent","momentum","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On Solving Minimax Optimization Locally: A Follow-the-Ridge Approach"},"forum":"Hkx7_1rKwS","id":"Hkx7_1rKwS"},{"content":{"TLDR":"Learning disentangled representations is considered a cornerstone problem in representation learning. Recently, Locatello et al. (2019) demonstrated that unsupervised disentanglement learning without inductive biases is theoretically impossible and t...","abstract":" ","authors":["Francesco Locatello","Michael Tschannen","Stefan Bauer","Gunnar R\u00e4tsch","Bernhard Sch\u00f6lkopf","Olivier Bachem"],"iclr_id":"SygagpEKwB","keywords":["disentanglement","inductive bias","representation learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Disentangling Factors of Variations Using Few Labels"},"forum":"SygagpEKwB","id":"SygagpEKwB"},{"content":{"TLDR":"Defending Against Physically Realizable Attacks on Image Classification","abstract":" ","authors":["Tong Wu","Liang Tong","Yevgeniy Vorobeychik"],"iclr_id":"H1xscnEKDr","keywords":["adversarial","adversarial machine learning","randomized smoothing","robustness"],"recs":[],"session":["Monday: Robustness and Verification"],"session_links":[null],"session_times":[null],"title":"Defending Against Physically Realizable Attacks on Image Classification"},"forum":"H1xscnEKDr","id":"H1xscnEKDr"},{"content":{"TLDR":"We propose a learned image-guided rendering technique that combines the benefits of image-based rendering and GAN-based image synthesis while considering view-dependent effects.","abstract":" ","authors":["Justus Thies","Michael Zollh\u00f6fer","Christian Theobalt","Marc Stamminger","Matthias Nie\u00dfner"],"iclr_id":"Hyg9anEFPS","keywords":["capacity"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Image-guided Neural Object Rendering"},"forum":"Hyg9anEFPS","id":"Hyg9anEFPS"},{"content":{"TLDR":"Method for addressing covariate shift in imitation learning using ensemble uncertainty","abstract":" ","authors":["Kiante Brantley","Wen Sun","Mikael Henaff"],"iclr_id":"rkgbYyHtwB","keywords":["adversarial","continuous control","imitation learning","reinforcement learning","uncertainty"],"recs":[],"session":["Monday: Reliable RL"],"session_links":[null],"session_times":[null],"title":"Disagreement-Regularized Imitation Learning"},"forum":"rkgbYyHtwB","id":"rkgbYyHtwB"},{"content":{"TLDR":"A method for learning to predict uncertainties over orientations using the Bingham Distribution","abstract":" ","authors":["Igor Gilitschenski","Roshni Sahoo","Wilko Schwarting","Alexander Amini","Sertac Karaman","Daniela Rus"],"iclr_id":"ryloogSKDS","keywords":["interpretability","reasoning","uncertainty"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep Orientation Uncertainty Learning based on a Bingham Loss"},"forum":"ryloogSKDS","id":"ryloogSKDS"},{"content":{"TLDR":"We present a new grid-particle learning method to process point clouds motivated by computational fluid dynamics.","abstract":" ","authors":["Xingzhe He","Helen Lu Cao","Bo Zhu"],"iclr_id":"H1eqQeHFDS","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"AdvectiveNet: An Eulerian-Lagrangian Fluidic Reservoir for Point Cloud Processing     "},"forum":"H1eqQeHFDS","id":"H1eqQeHFDS"},{"content":{"TLDR":"A scattering transform followed by supervised dictionary learning reaches a higher accuracy than AlexNet on ImageNet.","abstract":" ","authors":["John Zarka","Louis Thiry","Tomas Angles","Stephane Mallat"],"iclr_id":"SJxWS64FwH","keywords":["dictionary learning","imagenet","representation learning","sparse coding"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep Network Classification by Scattering and Homotopy Dictionary Learning"},"forum":"SJxWS64FwH","id":"SJxWS64FwH"},{"content":{"TLDR":"We propose a novel semantically-guided architecture for self-supervised monocular depth estimation","abstract":" ","authors":["Vitor Guizilini","Rui Hou","Jie Li","Rares Ambrus","Adrien Gaidon"],"iclr_id":"ByxT7TNFvH","keywords":["computer vision","representation learning","self supervised learning","semantic segmentation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Semantically-Guided Representation Learning for Self-Supervised Monocular Depth"},"forum":"ByxT7TNFvH","id":"ByxT7TNFvH"},{"content":{"TLDR":"The critical locus of the loss function of a neural network is determined by the geometry of the functional space and by the parameterization of this space by the network's weights. We introduce a natural distinction between pure critical points, whi...","abstract":" ","authors":["Matthew Trager","Kathl\u00e9n Kohn","Joan Bruna"],"iclr_id":"rkgOlCVYvB","keywords":["loss landscape"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Pure and Spurious Critical Points: a Geometric Study of Linear Networks"},"forum":"rkgOlCVYvB","id":"rkgOlCVYvB"},{"content":{"TLDR":"In this work, we point to a new connection between DNNs expressivity and Sharkovsky\u2019s Theorem from dynamical systems, that enables us to characterize the depth-width trade-offs of ReLU networks ","abstract":" ","authors":["Vaggos Chatziafratis","Sai Ganesh Nagarajan","Ioannis Panageas","Xiao Wang"],"iclr_id":"BJe55gBtvH","keywords":["relu networks"],"recs":[],"session":["Monday: Optimisation Theory"],"session_links":[null],"session_times":[null],"title":"Depth-Width Trade-offs for ReLU Networks via Sharkovsky's Theorem"},"forum":"BJe55gBtvH","id":"BJe55gBtvH"},{"content":{"TLDR":"We employ graph neural networks in the variational EM framework for efficient inference and learning of Markov Logic Networks.","abstract":" ","authors":["Yuyu Zhang","Xinshi Chen","Yuan Yang","Arun Ramamurthy","Bo Li","Yuan Qi","Le Song"],"iclr_id":"rJg76kStwH","keywords":["graph networks","reasoning","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Efficient Probabilistic Logic Reasoning with Graph Neural Networks"},"forum":"rJg76kStwH","id":"rJg76kStwH"},{"content":{"TLDR":"An learning-based approach for detecting and fixing bugs in Javascript","abstract":" ","authors":["Elizabeth Dinella","Hanjun Dai","Ziyang Li","Mayur Naik","Le Song","Ke Wang"],"iclr_id":"SJeqs6EFvB","keywords":["graph networks"],"recs":[],"session":["Monday: Robustness and Verification"],"session_links":[null],"session_times":[null],"title":"HOPPITY: LEARNING GRAPH TRANSFORMATIONS TO DETECT AND FIX BUGS IN PROGRAMS"},"forum":"SJeqs6EFvB","id":"SJeqs6EFvB"},{"content":{"TLDR":"Few-shot classification aims to recognize novel categories with only few labeled images in each class. Existing metric-based few-shot classification algorithms predict categories by comparing the feature embeddings of query images with those from a f...","abstract":" ","authors":["Hung-Yu Tseng","Hsin-Ying Lee","Jia-Bin Huang","Ming-Hsuan Yang"],"iclr_id":"SJl5Np4tPr","keywords":["generalization","imagenet"],"recs":[],"session":["Thursday: Continual Learning and Few Shot Learning"],"session_links":[null],"session_times":[null],"title":"Cross-Domain Few-Shot Classification via Learned Feature-Wise Transformation"},"forum":"SJl5Np4tPr","id":"SJl5Np4tPr"},{"content":{"TLDR":"A Multiflow Network is a dynamic architecture for domain adaptation that learns potentially different computational graphs per domain, so as to map them to a common representation where inference can be performed in a domain-agnostic fashion.","abstract":" ","authors":["R\u00f3ger Berm\u00fadez-Chac\u00f3n","Mathieu Salzmann","Pascal Fua"],"iclr_id":"rJxycxHKDS","keywords":["computer vision","domain adaptation","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Domain Adaptive Multibranch Networks"},"forum":"rJxycxHKDS","id":"rJxycxHKDS"},{"content":{"TLDR":"One of the biggest issues in deep learning theory is the generalization ability of networks with huge model size. \nThe classical learning theory suggests that overparameterized models cause overfitting.\nHowever, practically used large deep models avo...","abstract":" ","authors":["Taiji Suzuki","Hiroshi Abe","Tomoaki Nishimura"],"iclr_id":"ByeGzlrKwH","keywords":["compression","deep learning theory","generalization","generalization error","overfitting"],"recs":[],"session":["Wednesday: Theory"],"session_links":[null],"session_times":[null],"title":"Compression based bound for non-compressed network: unified generalization error analysis of large compressible deep neural network"},"forum":"ByeGzlrKwH","id":"ByeGzlrKwH"},{"content":{"TLDR":"We develop a principled theoretical framework for understanding and enforcing the stability of various types of GANs","abstract":" ","authors":["Casey Chu","Kentaro Minami","Kenji Fukumizu"],"iclr_id":"HJeOekHKwr","keywords":["adversarial","gan","gradient descent","stability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Smoothness and Stability in GANs"},"forum":"HJeOekHKwr","id":"HJeOekHKwr"},{"content":{"TLDR":"This paper proposed a novel framework for graph similarity learning in inductive and unsupervised scenario.","abstract":" ","authors":["Lichen Wang","Bo Zong","Qianqian Ma","Wei Cheng","Jingchao Ni","Wenchao Yu","Yanchi Liu","Dongjin Song","Haifeng Chen","Yun Fu"],"iclr_id":"rkem91rtDB","keywords":["graph isomorphism","graph networks","representation learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Inductive and Unsupervised Representation Learning on Graph Structured Objects"},"forum":"rkem91rtDB","id":"rkem91rtDB"},{"content":{"TLDR":"We propose an efficient algorithm for determining multilingually comparable sparse word representations that we release for 27 typologically diverse languages.","abstract":" ","authors":["G\u00e1bor Berend"],"iclr_id":"HyeYTgrFPB","keywords":["interpretability","nlp","sparse coding"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Massively Multilingual Sparse Word Representations"},"forum":"HyeYTgrFPB","id":"HyeYTgrFPB"},{"content":{"TLDR":"A multi-level spectral approach to improving the quality and scalability of unsupervised graph embedding.","abstract":" ","authors":["Chenhui Deng","Zhiqiang Zhao","Yongyu Wang","Zhiru Zhang","Zhuo Feng"],"iclr_id":"r1lGO0EKDH","keywords":["graph embedding","memory","optimization","scalability","unsupervised"],"recs":[],"session":["Thursday: Graphs and Representations"],"session_links":[null],"session_times":[null],"title":"GraphZoom: A Multi-level Spectral Approach for Accurate and Scalable Graph Embedding"},"forum":"r1lGO0EKDH","id":"r1lGO0EKDH"},{"content":{"TLDR":"Keras for infinite neural networks.","abstract":" ","authors":["Roman Novak","Lechao Xiao","Jiri Hron","Jaehoon Lee","Alexander A. Alemi","Jascha Sohl-Dickstein","Samuel S. Schoenholz"],"iclr_id":"SklD9yrFPS","keywords":["bayesian inference","distributed","gaussian process","gaussian processes","gradient descent","neural tangent kernel"],"recs":[],"session":["Tuesday: Probabilistic Approaches"],"session_links":[null],"session_times":[null],"title":"Neural Tangents: Fast and Easy Infinite Neural Networks in Python"},"forum":"SklD9yrFPS","id":"SklD9yrFPS"},{"content":{"TLDR":"We improve the sample-efficiency of the random search for functions defined on low-dimensional manifolds. Our method jointly learns the underlying manifold and optimizes the function.","abstract":" ","authors":["Ozan Sener","Vladlen Koltun"],"iclr_id":"B1gHokBKwS","keywords":["continuous control","derivative free optimization","optimization","search"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning to Guide Random Search"},"forum":"B1gHokBKwS","id":"B1gHokBKwS"},{"content":{"TLDR":"As deep neural networks (DNNs) achieve tremendous success across many application domains, researchers tried to explore in many aspects on why they generalize well. In this paper, we provide a novel perspective on these issues using the gradient sign...","abstract":" ","authors":["Jinlong Liu","Yunzhi Bai","Guoqing Jiang","Ting Chen","Huayan Wang"],"iclr_id":"HyevIJStwH","keywords":["dnn","generalization","gradient descent","optimization","regression"],"recs":[],"session":["Wednesday: Optimisation I"],"session_links":[null],"session_times":[null],"title":"Understanding Why Neural Networks Generalize Well Through GSNR of Parameters"},"forum":"HyevIJStwH","id":"HyevIJStwH"},{"content":{"TLDR":"We present a new routing method for Capsule networks, and it performs at-par with ResNet-18 on CIFAR-10/ CIFAR-100.","abstract":" ","authors":["Yao-Hung Hubert Tsai","Nitish Srivastava","Hanlin Goh","Ruslan Salakhutdinov"],"iclr_id":"HJe6uANtwH","keywords":["attention","capsule networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Capsules with Inverted Dot-Product Attention Routing"},"forum":"HJe6uANtwH","id":"HJe6uANtwH"},{"content":{"TLDR":"A feedforward layer to incorporate structured smoothness into a deep learning model","abstract":" ","authors":["Haroun Habeeb","Oluwasanmi Koyejo"],"iclr_id":"Hklr204Fvr","keywords":["inductive bias"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Towards a Deep Network Architecture for Structured Smoothness"},"forum":"Hklr204Fvr","id":"Hklr204Fvr"},{"content":{"TLDR":"Binary Neural Networks (BNNs) have been garnering interest thanks to their compute cost reduction and memory savings. However, BNNs suffer from performance degradation mainly due to the gradient mismatch caused by binarizing activations. Previous wor...","abstract":" ","authors":["Hyungjun Kim","Kyungsu Kim","Jinseok Kim","Jae-Joon Kim"],"iclr_id":"r1x0lxrFPS","keywords":["memory"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"BinaryDuo: Reducing Gradient Mismatch in Binary Activation Network by Coupling Binary Activations"},"forum":"r1x0lxrFPS","id":"r1x0lxrFPS"},{"content":{"TLDR":"Develop a general framework to establish certified robustness of ML models against various classes of adversarial perturbations","abstract":" ","authors":["Krishnamurthy (Dj) Dvijotham","Jamie Hayes","Borja Balle","Zico Kolter","Chongli Qin","Andras Gyorgy","Kai Xiao","Sven Gowal","Pushmeet Kohli"],"iclr_id":"SJlKrkSFPH","keywords":["adversarial","perturbation","randomized smoothing","robustness","verification"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A FRAMEWORK  FOR ROBUSTNESS CERTIFICATION  OF SMOOTHED CLASSIFIERS USING  F-DIVERGENCES"},"forum":"SJlKrkSFPH","id":"SJlKrkSFPH"},{"content":{"TLDR":"We propose a new certified adversarial training method, CROWN-IBP, that achieves state-of-the-art robustness for L_inf norm adversarial perturbations.","abstract":" ","authors":["Huan Zhang","Hongge Chen","Chaowei Xiao","Sven Gowal","Robert Stanforth","Bo Li","Duane Boning","Cho-Jui Hsieh"],"iclr_id":"Skxuk1rFwB","keywords":["adversarial","adversarial defense","interval bound propagation","perturbation","robustness","stability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Towards Stable and Efficient Training of Verifiably Robust Neural Networks"},"forum":"Skxuk1rFwB","id":"Skxuk1rFwB"},{"content":{"TLDR":"How to generate descriptions from structured data organized in tables? Existing approaches using neural encoder-decoder models often suffer from lacking diversity. We claim that an open set of templates is crucial for enriching the phrase constructio...","abstract":" ","authors":["Rong Ye","Wenxian Shi","Hao Zhou","Zhongyu Wei","Lei Li"],"iclr_id":"HkejNgBtPB","keywords":["generation","text generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Variational Template Machine for Data-to-Text Generation"},"forum":"HkejNgBtPB","id":"HkejNgBtPB"},{"content":{"TLDR":"A general method for computing the asymptotic behavior of wide networks using Feynman diagrams","abstract":" ","authors":["Ethan Dyer","Guy Gur-Ari"],"iclr_id":"S1gFvANKDS","keywords":["gradient descent"],"recs":[],"session":["Monday: Optimisation Theory"],"session_links":[null],"session_times":[null],"title":"Asymptotics of Wide Networks from Feynman Diagrams"},"forum":"S1gFvANKDS","id":"S1gFvANKDS"},{"content":{"TLDR":"(Frankle & Carbin, 2019) shows that there exist winning tickets (small but critical subnetworks) for dense, randomly initialized networks, that can be trained alone to achieve comparable accuracies to the latter in a similar number of iterations. How...","abstract":" ","authors":["Haoran You","Chaojian Li","Pengfei Xu","Yonggan Fu","Yue Wang","Xiaohan Chen","Richard G. Baraniuk","Zhangyang Wang","Yingyan Lin"],"iclr_id":"BJxsrgStvr","keywords":["learning rate"],"recs":[],"session":["Wednesday: Optimisation I"],"session_links":[null],"session_times":[null],"title":"Drawing Early-Bird Tickets: Toward More Efficient Training of Deep Networks"},"forum":"BJxsrgStvr","id":"BJxsrgStvr"},{"content":{"TLDR":"We identify and formalize the memorization problem in meta-learning and solve this problem with novel meta-regularization method, which greatly expand the domain that meta-learning can be  applicable to and effective on.","abstract":" ","authors":["Mingzhang Yin","George Tucker","Mingyuan Zhou","Sergey Levine","Chelsea Finn"],"iclr_id":"BklEFpEYwS","keywords":["information theory","memorization","meta learning","overfitting","regularization"],"recs":[],"session":["Monday: Meta-learning"],"session_links":[null],"session_times":[null],"title":"Meta-Learning without Memorization"},"forum":"BklEFpEYwS","id":"BklEFpEYwS"},{"content":{"TLDR":"We revisit self-training as a semi-supervised learning method for neural sequence generation problem, and show that self-training can be quite successful with injected noise.","abstract":" ","authors":["Junxian He","Jiatao Gu","Jiajun Shen","Marc'Aurelio Ranzato"],"iclr_id":"SJgdnAVKDH","keywords":["compositionality","dropout","generation","machine translation","perturbation","semi supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Revisiting Self-Training for Neural Sequence Generation"},"forum":"SJgdnAVKDH","id":"SJgdnAVKDH"},{"content":{"TLDR":"We propose a model that is able to perform physical parameter estimation of systems from video, where the differential equations governing the scene dynamics are known, but labeled states or objects are not available.","abstract":" ","authors":["Miguel Jaques","Michael Burke","Timothy Hospedales"],"iclr_id":"BJeKwTNFvB","keywords":["inductive bias","interpretability","reasoning","unsupervised","video prediction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Physics-as-Inverse-Graphics: Unsupervised Physical Parameter Estimation from Video"},"forum":"BJeKwTNFvB","id":"BJeKwTNFvB"},{"content":{"TLDR":"We consider the problem of translating, in an unsupervised manner, between two domains where one contains some additional information compared to the other. The proposed method  disentangles the common and separate parts of these domains and, through...","abstract":" ","authors":["Ron Mokady","Sagie Benaim","Lior Wolf","Amit Bermano"],"iclr_id":"BJe-91BtvH","keywords":["attention","generation","semantic segmentation","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Masked Based Unsupervised Content Transfer"},"forum":"BJe-91BtvH","id":"BJe-91BtvH"},{"content":{"TLDR":"FGSM-based adversarial training, with randomization, works just as well as PGD-based adversarial training: we can use this to train a robust classifier in 6 minutes on CIFAR10, and 12 hours on ImageNet, on a single machine.","abstract":" ","authors":["Eric Wong","Leslie Rice","J. Zico Kolter"],"iclr_id":"BJx040EFvH","keywords":["adversarial","imagenet"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Fast is better than free: Revisiting adversarial training"},"forum":"BJx040EFvH","id":"BJx040EFvH"},{"content":{"TLDR":"A data-driven learning algorithm based on unrolling the Alternating Minimization optimization for sparse graph recovery.","abstract":" ","authors":["Harsh Shrivastava","Xinshi Chen","Binghong Chen","Guanghui Lan","Srinivas Aluru","Han Liu","Le Song"],"iclr_id":"BkxpMTEtPB","keywords":["inductive bias","meta learning","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"GLAD: Learning Sparse Graph Recovery"},"forum":"BkxpMTEtPB","id":"BkxpMTEtPB"},{"content":{"TLDR":"We use ideas from quantum computing to propose word embeddings that utilize much fewer trainable parameters.","abstract":" ","authors":["Aliakbar Panahi","Seyran Saeedi","Tom Arodz"],"iclr_id":"HkxARkrFwB","keywords":["memory","nlp","word embedding","word embeddings"],"recs":[],"session":["Monday: Sequence Representations"],"session_links":[null],"session_times":[null],"title":"word2ket: Space-efficient Word Embeddings inspired by Quantum Entanglement"},"forum":"HkxARkrFwB","id":"HkxARkrFwB"},{"content":{"TLDR":"Robust models have robust feature extractors which can be useful for transferring robustness to other domains","abstract":" ","authors":["Ali Shafahi","Parsa Saadatpanah","Chen Zhu","Amin Ghiasi","Christoph Studer","David Jacobs","Tom Goldstein"],"iclr_id":"ryebG04YvB","keywords":["adversarial","fine tuning","generalization","lifelong learning","robustness","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Adversarially robust transfer learning"},"forum":"ryebG04YvB","id":"ryebG04YvB"},{"content":{"TLDR":"A method to explain a classifier, by generating visual perturbation of an image by exaggerating  or diminishing the semantic features that the classifier associates with a target label.","abstract":" ","authors":["Sumedha Singla","Brian Pollack","Junxiang Chen","Kayhan Batmanghelich"],"iclr_id":"H1xFWgrFPS","keywords":["black box","gan","interpretability"],"recs":[],"session":["Thursday: Fairness, Interpretabiity and Deployment"],"session_links":[null],"session_times":[null],"title":"Explanation  by Progressive  Exaggeration"},"forum":"H1xFWgrFPS","id":"H1xFWgrFPS"},{"content":{"TLDR":"Variational Autoencoders with latent spaces modeled as products of constant curvature Riemannian manifolds improve on image reconstruction over single-manifold variants.","abstract":" ","authors":["Ondrej Skopek","Octavian-Eugen Ganea","Gary B\u00e9cigneul"],"iclr_id":"S1g6xeSKDS","keywords":["autoencoder","generative models","variational autoencoders"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Mixed-curvature Variational Autoencoders"},"forum":"S1g6xeSKDS","id":"S1g6xeSKDS"},{"content":{"TLDR":"A study of how different components in the NAS pipeline contribute to the final accuracy. Also, a benchmark of 8 methods on 5 datasets.","abstract":" ","authors":["Antoine Yang","Pedro M. Esperan\u00e7a","Fabio M. Carlucci"],"iclr_id":"HygrdpVKvr","keywords":["cnn","neural architecture search","reproducibility"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"NAS evaluation is frustratingly hard"},"forum":"HygrdpVKvr","id":"HygrdpVKvr"},{"content":{"TLDR":"Efficient deep neural network (DNN) inference on mobile or embedded devices typically involves quantization of the network parameters and activations. In particular, mixed precision networks achieve better performance than networks with homogeneous b...","abstract":" ","authors":["Stefan Uhlich","Lukas Mauch","Fabien Cardinaux","Kazuki Yoshiyama","Javier Alonso Garcia","Stephen Tiedemann","Thomas Kemp","Akira Nakamura"],"iclr_id":"Hyx0slrFvH","keywords":["compression","imagenet","network compression","quantization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Mixed Precision DNNs: All you need is a good parametrization"},"forum":"Hyx0slrFvH","id":"Hyx0slrFvH"},{"content":{"TLDR":"We introduce Variational Intrinsic Successor FeatuRes (VISR), a novel algorithm which learns controllable features that can be leveraged to provide fast task inference through the successor features framework.","abstract":" ","authors":["Steven Hansen","Will Dabney","Andre Barreto","David Warde-Farley","Tom Van de Wiele","Volodymyr Mnih"],"iclr_id":"BJeAHkrYDS","keywords":["generalization","reinforcement learning","unsupervised"],"recs":[],"session":["Thursday: Continual Learning and Few Shot Learning"],"session_links":[null],"session_times":[null],"title":"Fast Task Inference with Variational Intrinsic Successor Features"},"forum":"BJeAHkrYDS","id":"BJeAHkrYDS"},{"content":{"TLDR":"Reinforcement learning formulation that allows agents to think and act at the same time, demonstrated on real-world robotic grasping.","abstract":" ","authors":["Ted Xiao","Eric Jang","Dmitry Kalashnikov","Sergey Levine","Julian Ibarz","Karol Hausman","Alexander Herzog"],"iclr_id":"SJexHkSFPS","keywords":["reinforcement learning","robotics"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Thinking While Moving: Deep Reinforcement Learning with Concurrent Control"},"forum":"SJexHkSFPS","id":"SJexHkSFPS"},{"content":{"TLDR":"we can significantly improve the computational efficiency of data selection in deep learning by using a much smaller proxy model to perform data selection.","abstract":" ","authors":["Cody Coleman","Christopher Yeh","Stephen Mussmann","Baharan Mirzasoleiman","Peter Bailis","Percy Liang","Jure Leskovec","Matei Zaharia"],"iclr_id":"HJg2b0VYDr","keywords":["active learning","imagenet","uncertainty"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Selection via Proxy: Efficient Data Selection for Deep Learning"},"forum":"HJg2b0VYDr","id":"HJg2b0VYDr"},{"content":{"TLDR":"Description of the limiting spectrum of the Hesian of the loss surface of DNNs in the infinite-width limit.","abstract":" ","authors":["Arthur Jacot","Franck Gabriel","Clement Hongler"],"iclr_id":"SkgscaNYPS","keywords":["deep learning theory","gradient descent","loss surface","neural tangent kernel"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"The asymptotic spectrum of the Hessian of DNN throughout training"},"forum":"SkgscaNYPS","id":"SkgscaNYPS"},{"content":{"TLDR":"IMPACT helps RL agents train faster by decreasing training wall-clock time and increasing sample efficiency simultaneously.","abstract":" ","authors":["Michael Luo","Jiahao Yao","Richard Liaw","Eric Liang","Ion Stoica"],"iclr_id":"BJeGlJStPr","keywords":["continuous control","distributed","importance sampling","reinforcement learning","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"IMPACT: Importance Weighted Asynchronous Architectures with Clipped Target Networks"},"forum":"BJeGlJStPr","id":"BJeGlJStPr"},{"content":{"TLDR":"We propose novel neural network architectures, guaranteed to satisfy the triangle inequality, for purposes of (asymmetric) metric learning and modeling graph distances. ","abstract":" ","authors":["Silviu Pitis","Harris Chan","Kiarash Jamali","Jimmy Ba"],"iclr_id":"HJeiDpVFPr","keywords":["inductive bias","metric learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"An Inductive Bias for Distances: Neural Nets that Respect the Triangle Inequality"},"forum":"HJeiDpVFPr","id":"HJeiDpVFPr"},{"content":{"TLDR":"Graph-based recurrent retriever that learns to retrieve reasoning paths over Wikipedia Graph outperforms the most recent state of the art on HotpotQA by more than 14 points.","abstract":" ","authors":["Akari Asai","Kazuma Hashimoto","Hannaneh Hajishirzi","Richard Socher","Caiming Xiong"],"iclr_id":"SJgVHkrYDH","keywords":["question answering","reasoning","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning to Retrieve Reasoning Paths over Wikipedia Graph for Question Answering"},"forum":"SJgVHkrYDH","id":"SJgVHkrYDH"},{"content":{"TLDR":"We provably recover the span of a deep multi-layered neural network with latent structure and empirically apply efficient span recovery algorithms to attack networks by obfuscating inputs.","abstract":" ","authors":["Rajesh Jayaram","David P. Woodruff","Qiuyi Zhang"],"iclr_id":"B1guLAVFDB","keywords":["adversarial"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Span Recovery for Deep Neural Networks with Applications to Input Obfuscation"},"forum":"B1guLAVFDB","id":"B1guLAVFDB"},{"content":{"TLDR":"DeFINE uses a deep, hierarchical, sparse network with new skip connections to learn better word embeddings efficiently. ","abstract":" ","authors":["Sachin Mehta","Rik Koncel-Kedziorski","Mohammad Rastegari","Hannaneh Hajishirzi"],"iclr_id":"rJeXS04FPH","keywords":["language modeling","machine translation","nlp","sequence modeling","transformer","word embedding"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"DeFINE: Deep Factorized Input Token Embeddings for Neural Sequence Modeling"},"forum":"rJeXS04FPH","id":"rJeXS04FPH"},{"content":{"TLDR":"One-shot neural architecture search (NAS) has played a crucial role in making\nNAS methods computationally feasible in practice. Nevertheless, there is still a\nlack of understanding on how these weight-sharing algorithms exactly work due\nto the many f...","abstract":" ","authors":["Arber Zela","Julien Siems","Frank Hutter"],"iclr_id":"SJx9ngStPH","keywords":["computer vision","neural architecture search"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"NAS-Bench-1Shot1: Benchmarking and Dissecting One-shot Neural Architecture Search"},"forum":"SJx9ngStPH","id":"SJx9ngStPH"},{"content":{"TLDR":"Recently, the pre-trained language model, BERT (and its robustly optimized version RoBERTa), has attracted a lot of attention in natural language understanding (NLU), and achieved state-of-the-art accuracy in various NLU tasks, such as sentiment clas...","abstract":" ","authors":["Wei Wang","Bin Bi","Ming Yan","Chen Wu","Jiangnan Xia","Zuyi Bao","Liwei Peng","Luo Si"],"iclr_id":"BJgQ4lSFPH","keywords":["attention","nlp","question answering"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"StructBERT: Incorporating Language Structures into Pre-training for Deep Language Understanding"},"forum":"BJgQ4lSFPH","id":"BJgQ4lSFPH"},{"content":{"TLDR":"Abductive reasoning is inference to the most plausible explanation. For example, if Jenny finds her house in a mess when she returns from work, and remembers that she left a window open, she can hypothesize that a thief broke into her house and  caus...","abstract":" ","authors":["Chandra Bhagavatula","Ronan Le Bras","Chaitanya Malaviya","Keisuke Sakaguchi","Ari Holtzman","Hannah Rashkin","Doug Downey","Wen-tau Yih","Yejin Choi"],"iclr_id":"Byg1v1HKDB","keywords":["generation","natural language generation","natural language inference","nlp","question answering","reasoning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Abductive Commonsense Reasoning"},"forum":"Byg1v1HKDB","id":"Byg1v1HKDB"},{"content":{"TLDR":"Settling permutation equivariance universality for popular deep models. ","abstract":" ","authors":["Nimrod Segol","Yaron Lipman"],"iclr_id":"HkxTwkrKDB","keywords":["equivariance"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On Universal Equivariant Set Networks"},"forum":"HkxTwkrKDB","id":"HkxTwkrKDB"},{"content":{"TLDR":"We propose a generative neural network approach for temporally coherent point clouds.","abstract":" ","authors":["Lukas Prantl","Nuttapong Chentanez","Stefan Jeschke","Nils Thuerey"],"iclr_id":"BJeKh3VYDH","keywords":["denoising"],"recs":[],"session":["Tuesday: Feature Discovery for Structured Data"],"session_links":[null],"session_times":[null],"title":"Tranquil Clouds: Neural Networks for Learning Temporally Coherent Features in Point Clouds"},"forum":"BJeKh3VYDH","id":"BJeKh3VYDH"},{"content":{"TLDR":"Allowing partial channel connection in super-networks to regularize and accelerate differentiable architecture search","abstract":" ","authors":["Yuhui Xu","Lingxi Xie","Xiaopeng Zhang","Xin Chen","Guo-Jun Qi","Qi Tian","Hongkai Xiong"],"iclr_id":"BJlS634tPr","keywords":["imagenet","memory","neural architecture search","normalization","regularization","stability","uncertainty"],"recs":[],"session":["Thursday: Network Architectures"],"session_links":[null],"session_times":[null],"title":"PC-DARTS: Partial Channel Connections for Memory-Efficient Architecture Search"},"forum":"BJlS634tPr","id":"BJlS634tPr"},{"content":{"TLDR":"\nTraining neural machine translation models (NMT) requires a large amount of parallel corpus, which is scarce for many language pairs. However, raw non-parallel corpora are often easy to obtain. Existing approaches have not exploited the full potenti...","abstract":" ","authors":["Zaixiang Zheng","Hao Zhou","Shujian Huang","Lei Li","Xin-Yu Dai","Jiajun Chen"],"iclr_id":"HkxQRTNYPH","keywords":["generative models","machine translation","nlp"],"recs":[],"session":["Thursday: Natural Language"],"session_links":[null],"session_times":[null],"title":"Mirror-Generative Neural Machine Translation"},"forum":"HkxQRTNYPH","id":"HkxQRTNYPH"},{"content":{"TLDR":"Algorithm for training individually fair classifier using adversarial robustness","abstract":" ","authors":["Mikhail Yurochkin","Amanda Bower","Yuekai Sun"],"iclr_id":"B1gdkxHFDH","keywords":["adversarial","fairness","optimization","perturbation","robustness"],"recs":[],"session":["Thursday: Fairness, Interpretabiity and Deployment"],"session_links":[null],"session_times":[null],"title":"Training individually fair ML models with sensitive subspace robustness"},"forum":"B1gdkxHFDH","id":"B1gdkxHFDH"},{"content":{"TLDR":"Don't deform your convolutions -- deform your kernels.","abstract":" ","authors":["Hang Gao","Xizhou Zhu","Stephen Lin","Jifeng Dai"],"iclr_id":"SkxSv6VFvS","keywords":["capacity","cnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deformable Kernels: Adapting Effective Receptive Fields for Object Deformation"},"forum":"SkxSv6VFvS","id":"SkxSv6VFvS"},{"content":{"TLDR":"We propose a deep generative model of volumes for 3D cryo-EM reconstruction from unlabelled 2D images and show that it can learn can learn continuous deformations in protein structure.","abstract":" ","authors":["Ellen D. Zhong","Tristan Bepler","Joseph H. Davis","Bonnie Berger"],"iclr_id":"SJxUjlBtwB","keywords":["3d reconstruction","clustering","ensembles","generative models","variational inference"],"recs":[],"session":["Tuesday: Biology and ML"],"session_links":[null],"session_times":[null],"title":"Reconstructing continuous distributions of 3D protein structure from cryo-EM images"},"forum":"SJxUjlBtwB","id":"SJxUjlBtwB"},{"content":{"TLDR":"Due to insufficient training data and the high computational cost to train a deep neural network from scratch, transfer learning has been extensively used in many deep-neural-network-based applications. A commonly used transfer learning approach invo...","abstract":" ","authors":["Shahbaz Rezaei","Xin Liu"],"iclr_id":"BylVcTNtDS","keywords":["security","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Target-Agnostic Attack on Deep Models: Exploiting Security Vulnerabilities of Transfer Learning"},"forum":"BylVcTNtDS","id":"BylVcTNtDS"},{"content":{"TLDR":"Using inducing point sparse Gaussian process methods to overcome catastrophic forgetting in neural networks.","abstract":" ","authors":["Michalis K. Titsias","Jonathan Schwarz","Alexander G. de G. Matthews","Razvan Pascanu","Yee Whye Teh"],"iclr_id":"HkxCzeHFDB","keywords":["bayesian inference","continual learning","distributed","gaussian process","gaussian processes","incremental learning","lifelong learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Functional Regularisation for  Continual Learning with Gaussian Processes"},"forum":"HkxCzeHFDB","id":"HkxCzeHFDB"},{"content":{"TLDR":"We propose an approach to learn sparse high dimensional representations that are fast to search, by incorporating a surrogate of the number of operations directly into the loss function.","abstract":" ","authors":["Biswajit Paria","Chih-Kuan Yeh","Ian E.H. Yen","Ning Xu","Pradeep Ravikumar","Barnab\u00e1s P\u00f3czos"],"iclr_id":"SygpC6Ntvr","keywords":["capacity","distributed","locality sensitive hashing","metric learning","quantization","regularization","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Minimizing FLOPs to Learn Efficient Sparse Representations"},"forum":"SygpC6Ntvr","id":"SygpC6Ntvr"},{"content":{"TLDR":"We learn particle-based fluid simulation with convolutional networks.","abstract":" ","authors":["Benjamin Ummenhofer","Lukas Prantl","Nils Thuerey","Vladlen Koltun"],"iclr_id":"B1lDoJSYDH","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Lagrangian Fluid Simulation with Continuous Convolutions"},"forum":"B1lDoJSYDH","id":"B1lDoJSYDH"},{"content":{"TLDR":"A new pretraining method that establishes new state-of-the-art results on the GLUE, RACE, and SQuAD benchmarks while having fewer parameters compared to BERT-large. ","abstract":" ","authors":["Zhenzhong Lan","Mingda Chen","Sebastian Goodman","Kevin Gimpel","Piyush Sharma","Radu Soricut"],"iclr_id":"H1eA7AEtvS","keywords":["memory","nlp","representation learning","self supervised learning","transformer"],"recs":[],"session":["Thursday: Natural Language"],"session_links":[null],"session_times":[null],"title":"ALBERT: A Lite BERT for Self-supervised Learning of Language Representations"},"forum":"H1eA7AEtvS","id":"H1eA7AEtvS"},{"content":{"TLDR":"Identifying when to give treatments to patients and how to select among multiple treatments over time are important medical problems with a few existing solutions. In this paper, we introduce the Counterfactual Recurrent Network (CRN), a novel sequen...","abstract":" ","authors":["Ioana Bica","Ahmed M Alaa","James Jordon","Mihaela van der Schaar"],"iclr_id":"BJg866NFvB","keywords":["adversarial","causal inference"],"recs":[],"session":["Wednesday: Actions and Counterfactuals"],"session_links":[null],"session_times":[null],"title":"Estimating counterfactual treatment outcomes over time through adversarially balanced representations"},"forum":"BJg866NFvB","id":"BJg866NFvB"},{"content":{"TLDR":" We propose a novel graph inference learning framework by building structure relations to infer unknown node labels from those labeled nodes in an end-to-end way.","abstract":" ","authors":["Chunyan Xu","Zhen Cui","Xiaobin Hong","Tong Zhang","Jian Yang","Wei Liu"],"iclr_id":"r1evOhEKvH","keywords":["optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Graph inference learning for semi-supervised classification"},"forum":"r1evOhEKvH","id":"r1evOhEKvH"},{"content":{"TLDR":"A modular and hierarchical approach to learn policies for exploring 3D environments.","abstract":" ","authors":["Devendra Singh Chaplot","Dhiraj Gandhi","Saurabh Gupta","Abhinav Gupta","Ruslan Salakhutdinov"],"iclr_id":"HklXn1BKDH","keywords":["navigation","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning To Explore Using Active Neural SLAM"},"forum":"HklXn1BKDH","id":"HklXn1BKDH"},{"content":{"TLDR":"Communication efficient federated learning with layer-wise matching","abstract":" ","authors":["Hongyi Wang","Mikhail Yurochkin","Yuekai Sun","Dimitris Papailiopoulos","Yasaman Khazaeni"],"iclr_id":"BkluqlSFDS","keywords":["cnn","federated learning"],"recs":[],"session":["Thursday: Fairness, Interpretabiity and Deployment"],"session_links":[null],"session_times":[null],"title":"Federated Learning with Matched Averaging"},"forum":"BkluqlSFDS","id":"BkluqlSFDS"},{"content":{"TLDR":"Humans in the loop revise documents to accord with counterfactual labels, resulting resource helps to reduce reliance on spurious associations.","abstract":" ","authors":["Divyansh Kaushik","Eduard Hovy","Zachary Lipton"],"iclr_id":"Sklgs0NFvr","keywords":["causality","natural language inference","nlp","sentiment analysis","text classification"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning The Difference That Makes A Difference With Counterfactually-Augmented Data"},"forum":"Sklgs0NFvr","id":"Sklgs0NFvr"},{"content":{"TLDR":"We propose a novel algorithm for learning fair representations that can simultaneously mitigate two notions of disparity among different demographic subgroups.","abstract":" ","authors":["Han Zhao","Amanda Coston","Tameem Adel","Geoffrey J. Gordon"],"iclr_id":"Hkekl0NFPr","keywords":["fairness","representation learning"],"recs":[],"session":["Thursday: Fairness, Interpretabiity and Deployment"],"session_links":[null],"session_times":[null],"title":"Conditional Learning of Fair Representations"},"forum":"Hkekl0NFPr","id":"Hkekl0NFPr"},{"content":{"TLDR":"An LSTM extension with state-of-the-art language modelling results.","abstract":" ","authors":["G\u00e1bor Melis","Tom\u00e1\u0161 Ko\u010disk\u00fd","Phil Blunsom"],"iclr_id":"SJe5P6EYvS","keywords":["generalization","memory","nlp","rnn","transformer"],"recs":[],"session":["Wednesday: Sequence Representations"],"session_links":[null],"session_times":[null],"title":"Mogrifier LSTM"},"forum":"SJe5P6EYvS","id":"SJe5P6EYvS"},{"content":{"TLDR":"We propose a generic neural network architecture equipping Pairwise Choice Markov Chains choice models with amortized and automatic differentiation based inference using alternatives' and individuals' features.","abstract":" ","authors":["Alix Lh\u00e9ritier"],"iclr_id":"BJgWE1SFwS","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"PCMC-Net: Feature-based Pairwise Choice Markov Chains"},"forum":"BJgWE1SFwS","id":"BJgWE1SFwS"},{"content":{"TLDR":"We perform efficient and flexible transfer learning in the framework of Bayesian optimization through meta-learned neural acquisition functions.","abstract":" ","authors":["Michael Volpp","Lukas P. Fr\u00f6hlich","Kirsten Fischer","Andreas Doerr","Stefan Falkner","Frank Hutter","Christian Daniel"],"iclr_id":"ryeYpJSKwr","keywords":["gaussian process","gaussian processes","generalization","meta learning","optimization","reinforcement learning","transfer learning"],"recs":[],"session":["Monday: Meta-learning"],"session_links":[null],"session_times":[null],"title":"Meta-Learning Acquisition Functions for Transfer Learning in Bayesian Optimization"},"forum":"ryeYpJSKwr","id":"ryeYpJSKwr"},{"content":{"TLDR":"This paper proposes DropEdge, a novel and flexible technique to alleviate over-smoothing and overfitting issue in deep Graph Convolutional Networks.","abstract":" ","authors":["Yu Rong","Wenbing Huang","Tingyang Xu","Junzhou Huang"],"iclr_id":"Hkx1qkrKPr","keywords":["cnn","generalization","graph networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"DropEdge: Towards Deep Graph Convolutional Networks on Node Classification"},"forum":"Hkx1qkrKPr","id":"Hkx1qkrKPr"},{"content":{"TLDR":"We enhance existing transformation-based defenses by using a distribution classifier on the distribution of softmax obtained from transformed images.","abstract":" ","authors":["Connie Kou","Hwee Kuan Lee","Ee-Chien Chang","Teck Khim Ng"],"iclr_id":"BkgWahEFvr","keywords":["adversarial","adversarial attacks","attention","cnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Enhancing Transformation-Based Defenses Against Adversarial Attacks with a Distribution Classifier"},"forum":"BkgWahEFvr","id":"BkgWahEFvr"},{"content":{"TLDR":"Posterior sampling for reinforcement learning (PSRL) is a useful framework for making decisions in an unknown environment.  PSRL maintains a posterior distribution of the environment and then makes planning on the environment sampled from the posteri...","abstract":" ","authors":["Yichi Zhou","Jialian Li","Jun Zhu"],"iclr_id":"Syg-ET4FPS","keywords":["multi agent reinforcement learning","planning","reinforcement learning"],"recs":[],"session":["Monday: Multiagent Systems"],"session_links":[null],"session_times":[null],"title":"Posterior sampling for multi-agent reinforcement learning: solving extensive games with imperfect information"},"forum":"Syg-ET4FPS","id":"Syg-ET4FPS"},{"content":{"TLDR":"We propose gated mechanisms to enhance learned ISTA for sparse coding, with theoretical guarantees on the superiority of the method. ","abstract":" ","authors":["Kailun Wu","Yiwen Guo","Ziang Li","Changshui Zhang"],"iclr_id":"BygPO2VKPH","keywords":["sparse coding"],"recs":[],"session":["Tuesday: Representation Learning"],"session_links":[null],"session_times":[null],"title":"Sparse Coding with Gated Learned ISTA"},"forum":"BygPO2VKPH","id":"BygPO2VKPH"},{"content":{"TLDR":"We propose a black-box algorithm called AdVIL  to perform inference and learning on a general Markov random field.","abstract":" ","authors":["Chongxuan Li","Chao Du","Kun Xu","Max Welling","Jun Zhu","Bo Zhang"],"iclr_id":"Sylgsn4Fvr","keywords":["adversarial","gradient descent","optimization","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"To Relieve Your Headache of Training an MRF, Take AdVIL"},"forum":"Sylgsn4Fvr","id":"Sylgsn4Fvr"},{"content":{"TLDR":"We proposed a Bayesian meta sampling method for adapting the model uncertainty in meta learning","abstract":" ","authors":["Zhenyi Wang","Yang Zhao","Ping Yu","Ruiyi Zhang","Changyou Chen"],"iclr_id":"Bkxv90EKPB","keywords":["meta learning","uncertainty","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Bayesian Meta Sampling for Fast Uncertainty Adaptation"},"forum":"Bkxv90EKPB","id":"Bkxv90EKPB"},{"content":{"TLDR":"We develop a strategy for pre-training Graph Neural Networks (GNNs) and systematically study its effectiveness on multiple datasets, GNN architectures, and diverse downstream tasks.","abstract":" ","authors":["Weihua Hu","Bowen Liu","Joseph Gomes","Marinka Zitnik","Percy Liang","Vijay Pande","Jure Leskovec"],"iclr_id":"HJlWWJSFDH","keywords":["generalization","graph networks","nlp","pre training","transfer learning"],"recs":[],"session":["Thursday: Graphs and Representations"],"session_links":[null],"session_times":[null],"title":"Strategies for Pre-training Graph Neural Networks"},"forum":"HJlWWJSFDH","id":"HJlWWJSFDH"},{"content":{"TLDR":"We scale up lossless compression with latent variables, achieving state of the art on full-size ImageNet images.","abstract":" ","authors":["James Townsend","Thomas Bird","Julius Kunze","David Barber"],"iclr_id":"r1lZgyBYwS","keywords":["compression","imagenet","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"HiLLoC: lossless image compression with hierarchical latent variable models"},"forum":"r1lZgyBYwS","id":"r1lZgyBYwS"},{"content":{"TLDR":"We show how to use data to automatically learn low-loss linkage procedures and metrics for specific clustering applications.","abstract":" ","authors":["Maria-Florina Balcan","Travis Dick","Manuel Lang"],"iclr_id":"S1eRbANtDB","keywords":["clustering","metric learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning to Link"},"forum":"S1eRbANtDB","id":"S1eRbANtDB"},{"content":{"TLDR":"Algorithm for imitation with state-only expert demonstrations; builds on adversarial-IRL; experiments with transition dynamics mismatch b/w expert and imitator","abstract":" ","authors":["Tanmay Gangwani","Jian Peng"],"iclr_id":"HJgLLyrYwB","keywords":["adversarial","imitation learning","inverse reinforcement learning","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"State-only Imitation with Transition Dynamics Mismatch"},"forum":"HJgLLyrYwB","id":"HJgLLyrYwB"},{"content":{"TLDR":"Imitation learning allows agents to learn complex behaviors from demonstrations. However, learning a complex vision-based task may require an impractical number of demonstrations. Meta-imitation learning is a promising approach towards enabling agent...","abstract":" ","authors":["Allan Zhou","Eric Jang","Daniel Kappler","Alex Herzog","Mohi Khansari","Paul Wohlhart","Yunfei Bai","Mrinal Kalakrishnan","Sergey Levine","Chelsea Finn"],"iclr_id":"SJg5J6NtDr","keywords":["imitation learning","meta learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Watch, Try, Learn: Meta-Learning from Demonstrations and Rewards"},"forum":"SJg5J6NtDr","id":"SJg5J6NtDr"},{"content":{"TLDR":"Policy gradient through backpropagation through time using learned models and Q-functions. SOTA results in reinforcement learning benchmark environments.","abstract":" ","authors":["Ignasi Clavera","Yao Fu","Pieter Abbeel"],"iclr_id":"Skln2A4YDB","keywords":["actor critic","model based reinforcement learning","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Model-Augmented Actor-Critic: Backpropagating through Paths"},"forum":"Skln2A4YDB","id":"Skln2A4YDB"},{"content":{"TLDR":"We use sparsity to improve the computational complexity of variance reduction methods.","abstract":" ","authors":["Melih Elibol","Lihua Lei","Michael I. Jordan"],"iclr_id":"Syx1DkSYwB","keywords":["nlp","optimization","variance reduction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Variance Reduction With Sparse Gradients"},"forum":"Syx1DkSYwB","id":"Syx1DkSYwB"},{"content":{"TLDR":"We compare the l4-norm based dictionary learning with PCA, ICA and show its stability as well as robustness.","abstract":" ","authors":["Yuexiang Zhai","Hermish Mehta","Zhengyuan Zhou","Yi Ma"],"iclr_id":"SJeY-1BKDS","keywords":["dictionary learning","robustness","sparse coding","stability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Understanding l4-based Dictionary Learning: Interpretation, Stability, and Robustness"},"forum":"SJeY-1BKDS","id":"SJeY-1BKDS"},{"content":{"TLDR":"We identify the security weakness of skip connections in ResNet-like neural networks","abstract":" ","authors":["Dongxian Wu","Yisen Wang","Shu-Tao Xia","James Bailey","Xingjun Ma"],"iclr_id":"BJlRs34Fvr","keywords":["adversarial","adversarial example","generation","security","transfer learning"],"recs":[],"session":["Monday: Robustness and Verification"],"session_links":[null],"session_times":[null],"title":"Skip Connections Matter: On the Transferability of Adversarial Examples Generated with ResNets"},"forum":"BJlRs34Fvr","id":"BJlRs34Fvr"},{"content":{"TLDR":"We propose MMA training to directly maximize input space margin in order to improve adversarial robustness primarily by removing the requirement of specifying a fixed distortion bound.","abstract":" ","authors":["Gavin Weiguang Ding","Yash Sharma","Kry Yik Chau Lui","Ruitong Huang"],"iclr_id":"HkeryxBtPB","keywords":["adversarial","perturbation","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"MMA Training: Direct Input Space Margin Maximization through Adversarial Training"},"forum":"HkeryxBtPB","id":"HkeryxBtPB"},{"content":{"TLDR":"We present a novel network pruning method that can find the optimal sparse structure during the training process with trainable pruning threshold","abstract":" ","authors":["Junjie LIU","Zhe XU","Runbin SHI","Ray C. C. Cheung","Hayden K.H. So"],"iclr_id":"SJlbGJrtDB","keywords":["compression","network compression","neural architecture search","optimization","pruning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Dynamic Sparse Training: Find Efficient Sparse Network From Scratch With Trainable Masked Layers"},"forum":"SJlbGJrtDB","id":"SJlbGJrtDB"},{"content":{"TLDR":"A method to automatically discover new categories in unlabelled data, by effectively transferring knowledge from labelled data of other different categories using feature rank statistics.","abstract":" ","authors":["Kai Han","Sylvestre-Alvise Rebuffi","Sebastien Ehrhardt","Andrea Vedaldi","Andrew Zisserman"],"iclr_id":"BJl2_nVFPB","keywords":["clustering","incremental learning","self supervised learning","semi supervised learning","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Automatically Discovering and Learning New Visual Categories with Ranking Statistics"},"forum":"BJl2_nVFPB","id":"BJl2_nVFPB"},{"content":{"TLDR":"First feedback alignment algorithm with provable learning guarantees for networks with single output neuron","abstract":" ","authors":["Mathias Lechner"],"iclr_id":"Bke61krFvS","keywords":["feedback alignment"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning representations for binary-classification without backpropagation"},"forum":"Bke61krFvS","id":"Bke61krFvS"},{"content":{"TLDR":"We propose a simple randomization technique for improving generalization in deep reinforcement learning across tasks with various unseen visual patterns.","abstract":" ","authors":["Kimin Lee","Kibok Lee","Jinwoo Shin","Honglak Lee"],"iclr_id":"HJgcvJBFvB","keywords":["generalization","regularization","reinforcement learning","robotics"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Network Randomization: A Simple Technique for Generalization in Deep Reinforcement Learning"},"forum":"HJgcvJBFvB","id":"HJgcvJBFvB"},{"content":{"TLDR":"We present a novel neural network compression method which can reuse the parameters efficiently to reduce the model size.","abstract":" ","authors":["Daquan Zhou","Xiaojie Jin","Qibin Hou","Kaixin Wang","Jianchao Yang","Jiashi Feng"],"iclr_id":"HyxjOyrKvr","keywords":["automl","compression","imagenet","knowledge distillation","model compression","network compression","pruning","quantization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Neural Epitome Search for Architecture-Agnostic Network Compression"},"forum":"HyxjOyrKvr","id":"HyxjOyrKvr"},{"content":{"TLDR":"A novel deep interpretable architecture that achieves state of the art on three large scale univariate time series forecasting datasets ","abstract":" ","authors":["Boris N. Oreshkin","Dmitri Carpov","Nicolas Chapados","Yoshua Bengio"],"iclr_id":"r1ecqn4YwB","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"N-BEATS: Neural basis expansion analysis for interpretable time series forecasting"},"forum":"r1ecqn4YwB","id":"r1ecqn4YwB"},{"content":{"TLDR":"We prove generalization bounds for convolutional neural networks that take account of weight-tying","abstract":" ","authors":["Philip M. Long","Hanie Sedghi"],"iclr_id":"r1e_FpNFDr","keywords":["cnn","deep learning theory","generalization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Generalization bounds for deep convolutional neural networks"},"forum":"r1e_FpNFDr","id":"r1e_FpNFDr"},{"content":{"TLDR":"This paper re-examines several common practices of setting hyper-parameters for fine-tuning and identify optimal hyperparameter depends on source-target domain similarity.","abstract":" ","authors":["Hao Li","Pratik Chaudhari","Hao Yang","Michael Lam","Avinash Ravichandran","Rahul Bhotika","Stefano Soatto"],"iclr_id":"B1g8VkHFPH","keywords":["computer vision","fine tuning","imagenet","learning rate","momentum","regularization","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Rethinking the Hyperparameters for Fine-tuning"},"forum":"B1g8VkHFPH","id":"B1g8VkHFPH"},{"content":{"TLDR":"We propose Choco-SGD---decentralized SGD with compressed communication---for non-convex objectives and show its strong performance in various deep learning applications (on-device learning, datacenter case).","abstract":" ","authors":["Anastasia Koloskova","Tao Lin","Sebastian U Stich","Martin Jaggi"],"iclr_id":"SkgGCkrKvH","keywords":["compression","privacy"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Decentralized Deep Learning with Arbitrary Communication Compression"},"forum":"SkgGCkrKvH","id":"SkgGCkrKvH"},{"content":{"TLDR":"Applying the softmax function in training leads to indirect and unexpected supervision on features. We propose a new training objective to explicitly induce dense feature regions for locally sufficient samples to benefit adversarial robustness.","abstract":" ","authors":["Tianyu Pang","Kun Xu","Yinpeng Dong","Chao Du","Ning Chen","Jun Zhu"],"iclr_id":"Byg9A24tvB","keywords":["adversarial","generalization","robust learning","robustness","trustworthy machine learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Rethinking Softmax Cross-Entropy Loss for Adversarial Robustness"},"forum":"Byg9A24tvB","id":"Byg9A24tvB"},{"content":{"TLDR":"We propose a link between permutation equivariance and compositional generalization, and provide equivariant language models","abstract":" ","authors":["Jonathan Gordon","David Lopez-Paz","Marco Baroni","Diane Bouchacourt"],"iclr_id":"SylVNerFvr","keywords":["compositionality","equivariance","generalization","language modeling","nlp","permutation equivariance"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Permutation Equivariant Models for Compositional Generalization in Language"},"forum":"SylVNerFvr","id":"SylVNerFvr"},{"content":{"TLDR":"The use of deep pre-trained transformers has led to remarkable progress in a number of applications (Devlin et al., 2018). For tasks that make pairwise comparisons between sequences, matching a given input with a corresponding label, two approaches a...","abstract":" ","authors":["Samuel Humeau","Kurt Shuster","Marie-Anne Lachaux","Jason Weston"],"iclr_id":"SkxgnnNFvH","keywords":["attention","fine tuning","self attention","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Poly-encoders: Architectures and Pre-training Strategies for Fast and Accurate Multi-sentence Scoring"},"forum":"SkxgnnNFvH","id":"SkxgnnNFvH"},{"content":{"TLDR":"Our work shows positional information has been implicitly encoded in a network. This information is important for detecting position-dependent features, e.g. semantic and saliency.","abstract":" ","authors":["Md Amirul Islam","Sen Jia","Neil D. B. Bruce"],"iclr_id":"rJeB36NKvB","keywords":["cnn"],"recs":[],"session":["Monday: Signals and Systems"],"session_links":[null],"session_times":[null],"title":"How much Position Information Do Convolutional Neural Networks Encode?"},"forum":"rJeB36NKvB","id":"rJeB36NKvB"},{"content":{"TLDR":"Machine learning has shown growing success in recent years. However, current machine learning systems are highly specialized, trained for particular problems or domains, and typically on a single narrow dataset. Human learning, on the other hand, is ...","abstract":" ","authors":["Emmanouil Antonios Platanios","Abulhair Saparov","Tom Mitchell"],"iclr_id":"Byx_YAVYPH","keywords":["reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Jelly Bean World: A Testbed for Never-Ending Learning"},"forum":"Byx_YAVYPH","id":"Byx_YAVYPH"},{"content":{"TLDR":"Transformer has become ubiquitous in natural language processing (e.g., machine translation, question answering); however, it requires enormous amount of computations to achieve high performance, which makes it not suitable for mobile applications si...","abstract":" ","authors":["Zhanghao Wu","Zhijian Liu","Ji Lin","Yujun Lin","Song Han"],"iclr_id":"ByeMPlHKPH","keywords":["attention","automl","compression","language modeling","machine translation","neural architecture search","nlp","question answering","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Lite Transformer with Long-Short Range Attention"},"forum":"ByeMPlHKPH","id":"ByeMPlHKPH"},{"content":{"TLDR":"We introduce a method to train models with provable robustness wrt all the $l_p$-norms for $p\\geq 1$ simultaneously.","abstract":" ","authors":["Francesco Croce","Matthias Hein"],"iclr_id":"rklk_ySYPB","keywords":["adversarial","adversarial attacks","perturbation","regularization","relu networks","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Provable robustness against all adversarial $l_p$-perturbations for $p\\geq 1$"},"forum":"rklk_ySYPB","id":"rklk_ySYPB"},{"content":{"TLDR":"This paper is about efficient Riemannian optimization on the Stiefel manifold that enforces the parameter matrices orthonormal.","abstract":" ","authors":["Jun Li","Fuxin Li","Sinisa Todorovic"],"iclr_id":"HJxV-ANKDH","keywords":["momentum","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Efficient Riemannian Optimization on the Stiefel Manifold via the Cayley Transform"},"forum":"HJxV-ANKDH","id":"HJxV-ANKDH"},{"content":{"TLDR":"We design an algorithm that reconstructs the key components of a novel deep learning system by exploiting a small amount of information leakage from a cache side-channel attack, Flush+Reload.","abstract":" ","authors":["Sanghyun Hong","Michael Davinroy","Yi\u01e7itcan Kaya","Dana Dachman-Soled","Tudor Dumitra\u015f"],"iclr_id":"S1erpeBFPB","keywords":["imagenet","neural architecture search","security"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"How to 0wn the NAS in Your Spare Time"},"forum":"S1erpeBFPB","id":"S1erpeBFPB"},{"content":{"TLDR":"Short-and-sparse deconvolution (SaSD) is the problem of extracting localized, recurring motifs in signals with spatial or temporal structure. Variants of this problem arise in applications such as image deblurring, microscopy, neural spike sorting, a...","abstract":" ","authors":["Yenson Lau","Qing Qu","Han-Wen Kuo","Pengcheng Zhou","Yuqian Zhang","John Wright"],"iclr_id":"Byg5ZANtvH","keywords":["acceleration","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Short and Sparse Deconvolution --- A Geometric Approach"},"forum":"Byg5ZANtvH","id":"Byg5ZANtvH"},{"content":{"TLDR":"SVQNs  formalizes the inference of hidden states and maximum entropy reinforcement learning under a unified graphical model and optimizes the two modules jointly.","abstract":" ","authors":["Shiyu Huang","Hang Su","Jun Zhu","Ting Chen"],"iclr_id":"r1xPh2VtPB","keywords":["generalization","generative models","reinforcement learning","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"SVQN: Sequential Variational Soft Q-Learning Networks"},"forum":"r1xPh2VtPB","id":"r1xPh2VtPB"},{"content":{"TLDR":"We propose a convergent proximal-type stochastic gradient descent algorithm for constrained nonsmooth nonconvex optimization problems","abstract":" ","authors":["Yang Yang","Yaxiong Yuan","Avraam Chatzimichailidis","Ruud JG van Sloun","Lei Lei","Symeon Chatzinotas"],"iclr_id":"HygpthEtvr","keywords":["gradient descent","learning rate","momentum","optimization","regularization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"ProxSGD: Training Structured Neural Networks under Regularization and Constraints"},"forum":"HygpthEtvr","id":"HygpthEtvr"},{"content":{"TLDR":"We introduce a novel version of Lipschitz objective continuity that allows stochastic mirror descent methodologies to achieve optimal convergence rates in problems with singularities.","abstract":" ","authors":["Kimon Antonakopoulos","E. Veronica Belmega","Panayotis Mertikopoulos"],"iclr_id":"rkxZyaNtwB","keywords":["optimization"],"recs":[],"session":["Monday: Optimisation Theory"],"session_links":[null],"session_times":[null],"title":"Online and stochastic optimization beyond Lipschitz continuity: A Riemannian approach"},"forum":"rkxZyaNtwB","id":"rkxZyaNtwB"},{"content":{"TLDR":"We evaluate self-supervised feature learning methods and find that with sufficient data augmentation early layers can be learned using just one image.  This is informative about self-supervision and the role of augmentations.","abstract":" ","authors":["Asano YM.","Rupprecht C.","Vedaldi A."],"iclr_id":"B1esx6EYvr","keywords":["cnn","representation learning","self supervision"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A critical analysis of self-supervision, or what we can learn from a single image"},"forum":"B1esx6EYvr","id":"B1esx6EYvr"},{"content":{"TLDR":"Tackling inverse design via genetic algorithms augmented with deep neural networks. ","abstract":" ","authors":["AkshatKumar Nigam","Pascal Friederich","Mario Krenn","Alan Aspuru-Guzik"],"iclr_id":"H1lmyRNFvr","keywords":["generative models","interpretability","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Augmenting Genetic Algorithms with Deep Neural Networks for Exploring the Chemical Space"},"forum":"H1lmyRNFvr","id":"H1lmyRNFvr"},{"content":{"TLDR":"Neural networks can approximate complex functions, but they struggle to perform exact arithmetic operations over real numbers. The lack of inductive bias for arithmetic operations leaves neural networks without the underlying logic necessary to extra...","abstract":" ","authors":["Andreas Madsen","Alexander Rosenberg Johansen"],"iclr_id":"H1gNOeHKPS","keywords":["inductive bias","generalization","inductive bias","interpretability","adversarial"],"recs":[],"session":["Thursday: Network Architectures"],"session_links":[null],"session_times":[null],"title":"Neural Arithmetic Units"},"forum":"H1gNOeHKPS","id":"H1gNOeHKPS"},{"content":{"TLDR":"A graph-based spherical CNN that strikes an interesting balance of trade-offs for a wide variety of applications.","abstract":" ","authors":["Micha\u00ebl Defferrard","Martino Milani","Fr\u00e9d\u00e9rick Gusset","Nathana\u00ebl Perraudin"],"iclr_id":"B1e3OlStPB","keywords":["equivariance","graph networks"],"recs":[],"session":["Thursday: Graphs and Representations"],"session_links":[null],"session_times":[null],"title":"DeepSphere: a graph-based spherical CNN"},"forum":"B1e3OlStPB","id":"B1e3OlStPB"},{"content":{"TLDR":"We extract a controllable model from a video of a person performing a certain activity. The model generates novel image sequences of that person, according to user-defined control signals, typically marking the displacement of the moving body. The ge...","abstract":" ","authors":["Oran Gafni","Lior Wolf","Yaniv Taigman"],"iclr_id":"SkxBUpEKwH","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Vid2Game: Controllable Characters Extracted from Real-World Videos"},"forum":"SkxBUpEKwH","id":"SkxBUpEKwH"},{"content":{"TLDR":"This paper presents how the loss surfaces of nonlinear neural networks are substantially shaped by the nonlinearities in activations.","abstract":" ","authors":["Fengxiang He","Bohan Wang","Dacheng Tao"],"iclr_id":"B1x6BTEKwr","keywords":["loss surface"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Piecewise linear activations substantially shape the loss surfaces of neural networks"},"forum":"B1x6BTEKwr","id":"B1x6BTEKwr"},{"content":{"TLDR":"We propose a self-ensemble framework to train more robust deep learning models under noisy labeled datasets.","abstract":" ","authors":["Duc Tam Nguyen","Chaithanya Kumar Mummadi","Thi Phuong Nhung Ngo","Thi Hoai Phuong Nguyen","Laura Beggel","Thomas Brox"],"iclr_id":"HkgsPhNYPS","keywords":["ensemble learning","noisy labels","robust learning","semi supervised learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"SELF: Learning to Filter Noisy Labels with Self-Ensembling"},"forum":"HkgsPhNYPS","id":"HkgsPhNYPS"},{"content":{"TLDR":"A weakly supervised learning based clustering framework performs comparable to that of fully supervised learning models by exploiting unique class count.","abstract":" ","authors":["Mustafa Umit Oner","Hwee Kuan Lee","Wing-Kin Sung"],"iclr_id":"B1xIj3VYvr","keywords":["clustering","semantic segmentation","weakly supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Weakly Supervised Clustering by Exploiting Unique Class Count"},"forum":"B1xIj3VYvr","id":"B1xIj3VYvr"},{"content":{"TLDR":" In this work, we present a method that implements an efficient backward pass through blackbox implementations of combinatorial solvers with linear objective functions.","abstract":" ","authors":["Marin Vlastelica Pogan\u010di\u0107","Anselm Paulus","Vit Musil","Georg Martius","Michal Rolinek"],"iclr_id":"BkevoJSYPB","keywords":["optimization","planning","representation learning","robotics"],"recs":[],"session":["Thursday: Optimisation II"],"session_links":[null],"session_times":[null],"title":"Differentiation of Blackbox Combinatorial Solvers"},"forum":"BkevoJSYPB","id":"BkevoJSYPB"},{"content":{"TLDR":"We present the first object-centric generative model of 3D visual scenes capable of both decomposing and generating scenes.","abstract":" ","authors":["Martin Engelcke","Adam R. Kosiorek","Oiwi Parker Jones","Ingmar Posner"],"iclr_id":"BkxfaTVFwH","keywords":["generation","generative models","reinforcement learning","robotics","semi supervised learning","unsupervised","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"GENESIS: Generative Scene Inference and Sampling with Object-Centric Latent Representations"},"forum":"BkxfaTVFwH","id":"BkxfaTVFwH"},{"content":{"TLDR":"how to achieve state-of-the-art performance by combining policy network in model-based planning","abstract":" ","authors":["Tingwu Wang","Jimmy Ba"],"iclr_id":"H1exf64KwH","keywords":["model based reinforcement learning","optimization","planning","reinforcement learning","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Exploring Model-based Planning with Policy Networks"},"forum":"H1exf64KwH","id":"H1exf64KwH"},{"content":{"TLDR":"Learning to cooperate is crucially important in multi-agent environments. The key is to understand the mutual interplay between agents. However, multi-agent environments are highly dynamic, where agents keep moving and their neighbors change quickly....","abstract":" ","authors":["Jiechuan Jiang","Chen Dun","Tiejun Huang","Zongqing Lu"],"iclr_id":"HkxdQkSYDB","keywords":["regularization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Graph Convolutional Reinforcement Learning"},"forum":"HkxdQkSYDB","id":"HkxdQkSYDB"},{"content":{"TLDR":"We classify mouse V1 neurons into putative functional cell types based on their representations in a CNN predicting neural responses","abstract":" ","authors":["Ivan Ustyuzhaninov","Santiago A. Cadena","Emmanouil Froudarakis","Paul G. Fahey","Edgar Y. Walker","Erick Cobos","Jacob Reimer","Fabian H. Sinz","Andreas S. Tolias","Matthias Bethge","Alexander S. Ecker"],"iclr_id":"rklr9kHFDB","keywords":["clustering","computational neuroscience","equivariance","visual cortex"],"recs":[],"session":["Tuesday: Biology and ML"],"session_links":[null],"session_times":[null],"title":"Rotation-invariant clustering of neuronal responses in primary visual cortex"},"forum":"rklr9kHFDB","id":"rklr9kHFDB"},{"content":{"TLDR":"Regularization and normalization have become indispensable components in training deep neural networks, resulting in faster training and improved generalization performance. We propose the projected error function regularization loss (PER) that encou...","abstract":" ","authors":["Taejong Joo","Donggu Kang","Byunghoon Kim"],"iclr_id":"rygwLgrYPB","keywords":["generalization","language modeling","nlp","regularization","wasserstein metric"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Regularizing activations in neural networks via distribution matching with the Wasserstein metric"},"forum":"rygwLgrYPB","id":"rygwLgrYPB"},{"content":{"TLDR":"We find that the lottery ticket phenomenon is present in both NLP and RL, and find that it can be used to train compressed Transformers to high performance","abstract":" ","authors":["Haonan Yu","Sergey Edunov","Yuandong Tian","Ari S. Morcos"],"iclr_id":"S1xnXRVFwH","keywords":["lottery tickets","nlp","optimization","pruning","reinforcement learning","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Playing the lottery with rewards and multiple languages: lottery tickets in RL and NLP"},"forum":"S1xnXRVFwH","id":"S1xnXRVFwH"},{"content":{"TLDR":"We find that existing bonus-based exploration methods have not been able to address the exploration-exploitation trade-off in the Arcade Learning Environment. ","abstract":" ","authors":["Adrien Ali Taiga","William Fedus","Marlos C. Machado","Aaron Courville","Marc G. Bellemare"],"iclr_id":"BJewlyStDr","keywords":["reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On Bonus Based Exploration Methods In The Arcade Learning Environment"},"forum":"BJewlyStDr","id":"BJewlyStDr"},{"content":{"TLDR":"Efficient Transformer with locality-sensitive hashing and reversible layers","abstract":" ","authors":["Nikita Kitaev","Lukasz Kaiser","Anselm Levskaya"],"iclr_id":"rkgNKkHtvB","keywords":["attention","locality sensitive hashing","memory","transformer"],"recs":[],"session":["Wednesday: Sequence Representations"],"session_links":[null],"session_times":[null],"title":"Reformer: The Efficient Transformer"},"forum":"rkgNKkHtvB","id":"rkgNKkHtvB"},{"content":{"TLDR":"We present a sign-based, rather than magnitude-based, gradient estimation approach that shifts gradient estimation from continuous to binary black-box optimization.","abstract":" ","authors":["Abdullah Al-Dujaili","Una-May O'Reilly"],"iclr_id":"SygW0TEFwH","keywords":["adversarial","imagenet","memory","optimization","zeroth order optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Sign Bits Are All You Need for Black-Box Attacks"},"forum":"SygW0TEFwH","id":"SygW0TEFwH"},{"content":{"TLDR":"We are proposing a new score-based approach to structure/causal learning leveraging neural networks and a recent continuous constrained formulation to this problem","abstract":" ","authors":["S\u00e9bastien Lachapelle","Philippe Brouillard","Tristan Deleu","Simon Lacoste-Julien"],"iclr_id":"rklbKA4YDS","keywords":["causality","density estimation","optimization","structure learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Gradient-Based Neural DAG Learning"},"forum":"rklbKA4YDS","id":"rklbKA4YDS"},{"content":{"TLDR":"Mini-batch stochastic gradient methods (SGD) are state of the art for distributed training of deep neural networks. \nDrastic increases in the mini-batch sizes have lead to key efficiency and scalability gains in recent years. \nHowever, progress faces...","abstract":" ","authors":["Tao Lin","Sebastian U. Stich","Kumar Kshitij Patel","Martin Jaggi"],"iclr_id":"B1eyO1BFPr","keywords":["distributed","generalization","scalability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Don't Use Large Mini-batches, Use Local SGD"},"forum":"B1eyO1BFPr","id":"B1eyO1BFPr"},{"content":{"TLDR":"We built a physical simulation of a rodent, trained it to solve a set of tasks, and analyzed the resulting networks.","abstract":" ","authors":["Josh Merel","Diego Aldarondo","Jesse Marshall","Yuval Tassa","Greg Wayne","Bence Olveczky"],"iclr_id":"SyxrxR4KPS","keywords":["computational neuroscience","reinforcement learning"],"recs":[],"session":["Tuesday: Biology and ML"],"session_links":[null],"session_times":[null],"title":"Deep neuroethology of a virtual rodent"},"forum":"SyxrxR4KPS","id":"SyxrxR4KPS"},{"content":{"TLDR":"For graph neural networks, the aggregation on a graph can benefit from a continuous space underlying the graph.","abstract":" ","authors":["Hongbin Pei","Bingzhe Wei","Kevin Chen-Chuan Chang","Yu Lei","Bo Yang"],"iclr_id":"S1e2agrFvS","keywords":["cnn","graph networks"],"recs":[],"session":["Thursday: Graphs and Representations"],"session_links":[null],"session_times":[null],"title":"Geom-GCN: Geometric Graph Convolutional Networks"},"forum":"S1e2agrFvS","id":"S1e2agrFvS"},{"content":{"TLDR":"We propose an exponentially growing learning rate schedule for networks with BatchNorm, which surprisingly performs well in practice and is provably equivalent to popular LR schedules like Step Decay.","abstract":" ","authors":["Zhiyuan Li","Sanjeev Arora"],"iclr_id":"rJg8TeSFDH","keywords":["batch normalization","deep learning theory","generalization","learning rate","momentum","optimization"],"recs":[],"session":["Wednesday: Optimisation I"],"session_links":[null],"session_times":[null],"title":"An Exponential Learning Rate Schedule for Deep Learning"},"forum":"rJg8TeSFDH","id":"rJg8TeSFDH"},{"content":{"TLDR":"The solution of gradient descent based adversarial training converges in direction to a robust max margin solution that is adapted to adversary geometry, using L2 perturbation also shows significant speed-up in convergence compared to clean training.","abstract":" ","authors":["Yan Li","Ethan X.Fang","Huan Xu","Tuo Zhao"],"iclr_id":"HkgTTh4FDH","keywords":["adversarial","gradient descent","implicit bias","perturbation","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Implicit Bias of Gradient Descent based Adversarial Training on Separable Data"},"forum":"HkgTTh4FDH","id":"HkgTTh4FDH"},{"content":{"TLDR":"We introduce unrestricted perturbations that manipulate semantically meaningful image-based visual descriptors - color and texture - in order to generate effective and  photorealistic adversarial examples.","abstract":" ","authors":["Anand Bhattad","Min Jin Chong","Kaizhao Liang","Bo Li","D. A. Forsyth"],"iclr_id":"Sye_OgHFwH","keywords":["adversarial","compression","imagenet","perturbation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Unrestricted Adversarial Examples via Semantic Manipulation"},"forum":"Sye_OgHFwH","id":"Sye_OgHFwH"},{"content":{"TLDR":"This paper shows that differential privacy could improve the utility of outlier detection, novelty detection and backdoor attack detection, through both a theoretical analysis and extensive experimental results (constructed and real-world).","abstract":" ","authors":["Min Du","Ruoxi Jia","Dawn Song"],"iclr_id":"SJx0q1rtvS","keywords":["anomaly detection","novelty detection","outlier detection","privacy"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Robust anomaly detection and backdoor attack detection via differential privacy"},"forum":"SJx0q1rtvS","id":"SJx0q1rtvS"},{"content":{"TLDR":"We study the impact of training strategies on the performance of knowledge graph embeddings.","abstract":" ","authors":["Daniel Ruffinelli","Samuel Broscheit","Rainer Gemulla"],"iclr_id":"BkxSmlBFvr","keywords":["graph embedding","knowledge graph embeddings","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"You CAN Teach an Old Dog New Tricks! On Training Knowledge Graph Embeddings"},"forum":"BkxSmlBFvr","id":"BkxSmlBFvr"},{"content":{"TLDR":"Layerdrop, a form of structured dropout that allows you to train one model at training time and prune to any desired depth at test time. You can also use this to train even deeper models.","abstract":" ","authors":["Angela Fan","Edouard Grave","Armand Joulin"],"iclr_id":"SylO2yStDr","keywords":["dropout","language modeling","machine translation","nlp","overfitting","pruning","question answering","regularization","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Reducing Transformer Depth on Demand with Structured Dropout"},"forum":"SylO2yStDr","id":"SylO2yStDr"},{"content":{"TLDR":"We show that training classifiers to produce salient input Jacobian matrices with a GAN-like regularization can boost adversarial robustness.","abstract":" ","authors":["Alvin Chan","Yi Tay","Yew Soon Ong","Jie Fu"],"iclr_id":"Hke0V1rKPS","keywords":["adversarial","perturbation","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Jacobian Adversarially Regularized Networks for Robustness"},"forum":"Hke0V1rKPS","id":"Hke0V1rKPS"},{"content":{"TLDR":"Adversarial patch attacks are among one of the most practical threat models against real-world computer vision systems. This paper studies certified and empirical defenses against patch attacks. We begin with a set of experiments showing that most ex...","abstract":" ","authors":["Ping-yeh Chiang","Renkun Ni","Ahmed Abdelkader","Chen Zhu","Christoph Studor","Tom Goldstein"],"iclr_id":"HyeaSkrYPH","keywords":["adversarial","computer vision","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Certified Defenses for Adversarial Patches"},"forum":"HyeaSkrYPH","id":"HyeaSkrYPH"},{"content":{"TLDR":"We have developed a new training algorithm for energy-based latent variable models that completely removes the bias of contrastive divergence.","abstract":" ","authors":["Yixuan Qiu","Lingsong Zhang","Xiao Wang"],"iclr_id":"r1eyceSYPr","keywords":[],"recs":[],"session":["Tuesday: Probabilistic Approaches"],"session_links":[null],"session_times":[null],"title":"Unbiased Contrastive Divergence Algorithm for Training Energy-Based Latent Variable Models"},"forum":"r1eyceSYPr","id":"r1eyceSYPr"},{"content":{"TLDR":"We apply the actor-critic methodology from reinforcement learning to collaborative filtering, resulting in improved performance across a variety of latent-variable models","abstract":" ","authors":["Sam Lobel","Chunyuan Li","Jianfeng Gao","Lawrence Carin"],"iclr_id":"HJxR7R4FvS","keywords":["actor critic","collaborative filtering","optimization","recommender systems","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"RaCT: Toward Amortized Ranking-Critical Training For Collaborative Filtering "},"forum":"HJxR7R4FvS","id":"HJxR7R4FvS"},{"content":{"TLDR":"We study the interplay between memorization and generalization of\noverparameterized networks in the extreme case of a single training example and an identity-mapping task. We examine fully-connected and convolutional networks (FCN and CNN), both line...","abstract":" ","authors":["Chiyuan Zhang","Samy Bengio","Moritz Hardt","Michael C. Mozer","Yoram Singer"],"iclr_id":"B1l6y0VFPr","keywords":["cnn","generalization","inductive bias","memorization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Identity Crisis: Memorization and Generalization Under Extreme Overparameterization"},"forum":"B1l6y0VFPr","id":"B1l6y0VFPr"},{"content":{"TLDR":"This work proposes an autoencoder with a novel robust subspace recovery layer for unsupervised anomaly detection and demonstrates state-of-the-art results on various datasets.","abstract":" ","authors":["Chieh-Hsin Lai","Dongmian Zou","Gilad Lerman"],"iclr_id":"rylb3eBtwr","keywords":["anomaly detection","autoencoder","unsupervised","unsupervised anomaly detection"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Robust Subspace Recovery Layer for Unsupervised Anomaly Detection"},"forum":"rylb3eBtwr","id":"rylb3eBtwr"},{"content":{"TLDR":"We show that removing constant terms from CNN architectures ensures strong generalization across noise levels, and also provides interpretability of the denoising method via linear-algebra techniques.","abstract":" ","authors":["Sreyas Mohan","Zahra Kadkhodaie","Eero P. Simoncelli","Carlos Fernandez-Granda"],"iclr_id":"HJlSmC4FPS","keywords":["cnn","denoising","generalization","interpretability","overfitting","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Robust And Interpretable Blind Image Denoising Via Bias-Free Convolutional Neural Networks"},"forum":"HJlSmC4FPS","id":"HJlSmC4FPS"},{"content":{"TLDR":"    We study how the behavior of deep policy gradient algorithms reflects the conceptual framework motivating their development. To this end, we propose a fine-grained analysis of state-of-the-art methods based on key elements of this framework: grad...","abstract":" ","authors":["Andrew Ilyas","Logan Engstrom","Shibani Santurkar","Dimitris Tsipras","Firdaus Janoos","Larry Rudolph","Aleksander Madry"],"iclr_id":"ryxdEkHtPS","keywords":["optimization","policy gradient","reinforcement learning"],"recs":[],"session":["Monday: Reliable RL"],"session_links":[null],"session_times":[null],"title":"A Closer Look at Deep Policy Gradients"},"forum":"ryxdEkHtPS","id":"ryxdEkHtPS"},{"content":{"TLDR":"We introduce GAN-TTS, a Generative Adversarial Network for Text-to-Speech, which achieves Mean Opinion Score (MOS) 4.2.","abstract":" ","authors":["Miko\u0142aj Bi\u0144kowski","Jeff Donahue","Sander Dieleman","Aidan Clark","Erich Elsen","Norman Casagrande","Luis C. Cobo","Karen Simonyan"],"iclr_id":"r1gfQgSFDr","keywords":["adversarial","attention","autoregressive models","gan","generative models"],"recs":[],"session":["Monday: Signals and Systems"],"session_links":[null],"session_times":[null],"title":"High Fidelity Speech Synthesis with Adversarial Networks"},"forum":"r1gfQgSFDr","id":"r1gfQgSFDr"},{"content":{"TLDR":"We introduce Distribution Matching and Augmentation Anchoring, two improvements to MixMatch which produce state-of-the-art results and enable surprisingly strong performance with only 40 labels on CIFAR-10 and SVHN.","abstract":" ","authors":["David Berthelot","Nicholas Carlini","Ekin D. Cubuk","Alex Kurakin","Kihyuk Sohn","Han Zhang","Colin Raffel"],"iclr_id":"HklkeR4KPB","keywords":["semi supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"ReMixMatch: Semi-Supervised Learning with Distribution Matching and Augmentation Anchoring"},"forum":"HklkeR4KPB","id":"HklkeR4KPB"},{"content":{"TLDR":"The long-tail distribution of the visual world poses great challenges for deep learning based classification models on how to handle the class imbalance problem. Existing solutions usually involve class-balancing strategies, e.g., by loss re-weightin...","abstract":" ","authors":["Bingyi Kang","Saining Xie","Marcus Rohrbach","Zhicheng Yan","Albert Gordo","Jiashi Feng","Yannis Kalantidis"],"iclr_id":"r1gRTCVFvB","keywords":["imagenet","memory","representation learning","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Decoupling Representation and Classifier for Long-Tailed Recognition"},"forum":"r1gRTCVFvB","id":"r1gRTCVFvB"},{"content":{"TLDR":"Formal analysis of  Binarized Neural Networks ","abstract":" ","authors":["Nina Narodytska","Hongce Zhang","Aarti Gupta","Toby Walsh"],"iclr_id":"SJx-j64FDr","keywords":["generation","reasoning","verification"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"In Search for a SAT-friendly Binarized Neural Network Architecture"},"forum":"SJx-j64FDr","id":"SJx-j64FDr"},{"content":{"TLDR":"Artificial neural networks suffer from catastrophic forgetting when they are sequentially trained on multiple tasks. To overcome this problem, we present a novel approach based on task-conditioned hypernetworks, i.e., networks that generate the weigh...","abstract":" ","authors":["Johannes von Oswald","Christian Henning","Jo\u00e3o Sacramento","Benjamin F. Grewe"],"iclr_id":"SJgwNerKvB","keywords":["capacity","catastrophic forgetting","continual learning","hypernetworks","memory","transfer learning"],"recs":[],"session":["Thursday: Continual Learning and Few Shot Learning","Thursday: Continual Learning and Few Shot Learning"],"session_links":[null,null],"session_times":[null,null],"title":"Continual learning with hypernetworks"},"forum":"SJgwNerKvB","id":"SJgwNerKvB"},{"content":{"TLDR":"We propose a self-adversarial learning (SAL) paradigm which improves the generator in a self-play fashion for improving GANs' performance in text generation.","abstract":" ","authors":["Wangchunshu Zhou","Tao Ge","Ke Xu","Furu Wei","Ming Zhou"],"iclr_id":"B1l8L6EtDS","keywords":["adversarial","gan","generation","text generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Self-Adversarial Learning with Comparative Discrimination for Text Generation"},"forum":"B1l8L6EtDS","id":"B1l8L6EtDS"},{"content":{"TLDR":"Adversarial training, which minimizes the maximal risk for label-preserving input perturbations, has proved to be effective for improving the generalization of language models. In this work, we propose a novel adversarial training algorithm, FreeLB, ...","abstract":" ","authors":["Chen Zhu","Yu Cheng","Zhe Gan","Siqi Sun","Tom Goldstein","Jingjing Liu"],"iclr_id":"BygzbyHFvB","keywords":["adversarial","generalization","nlp","perturbation","reasoning","transformer","word embedding","word embeddings"],"recs":[],"session":["Thursday: Natural Language"],"session_links":[null],"session_times":[null],"title":"FreeLB: Enhanced Adversarial Training for Natural Language Understanding"},"forum":"BygzbyHFvB","id":"BygzbyHFvB"},{"content":{"TLDR":"A state-value function-based version of MPO that achieves good results in a wide range of tasks in discrete and continuous control.","abstract":" ","authors":["H. Francis Song","Abbas Abdolmaleki","Jost Tobias Springenberg","Aidan Clark","Hubert Soyer","Jack W. Rae","Seb Noury","Arun Ahuja","Siqi Liu","Dhruva Tirumala","Nicolas Heess","Dan Belov","Martin Riedmiller","Matthew M. Botvinick"],"iclr_id":"SylOlp4FvH","keywords":["continuous control","multi task learning","optimization","policy gradient","regularization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"V-MPO: On-Policy Maximum a Posteriori Policy Optimization for Discrete and Continuous Control"},"forum":"SylOlp4FvH","id":"SylOlp4FvH"},{"content":{"TLDR":"Reinforcment practices for machine translation performance gains might not come from better predictions.","abstract":" ","authors":["Leshem Choshen","Lior Fox","Zohar Aizenbud","Omri Abend"],"iclr_id":"H1eCw3EKvH","keywords":["adversarial","gan","generation","machine translation","reinforcement learning","text generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On the Weaknesses of Reinforcement Learning for Neural Machine Translation"},"forum":"H1eCw3EKvH","id":"H1eCw3EKvH"},{"content":{"TLDR":"We propose to tackle complex tasks of multiple agents by learning composable primitive skills and coordination of the skills. ","abstract":" ","authors":["Youngwoon Lee","Jingyun Yang","Joseph J. Lim"],"iclr_id":"ryxB2lBtvH","keywords":["hierarchical reinforcement learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning to Coordinate Manipulation Skills via Skill Behavior Diversification"},"forum":"ryxB2lBtvH","id":"ryxB2lBtvH"},{"content":{"TLDR":"Models of source code that combine global and structural features learn more powerful representations of programs.","abstract":" ","authors":["Vincent J. Hellendoorn","Charles Sutton","Rishabh Singh","Petros Maniatis","David Bieber"],"iclr_id":"B1lnbRNtwr","keywords":["attention","distributed","graph networks","inductive bias","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Global Relational Models of Source Code"},"forum":"B1lnbRNtwr","id":"B1lnbRNtwr"},{"content":{"TLDR":"We provide the first algorithm for quantum computers implementing universal convolutional neural network with a speedup","abstract":" ","authors":["Iordanis Kerenidis","Jonas Landman","Anupam Prakash"],"iclr_id":"Hygab1rKDS","keywords":["cnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Quantum Algorithms for Deep Convolutional Neural Networks"},"forum":"Hygab1rKDS","id":"Hygab1rKDS"},{"content":{"TLDR":"Comprehensive analysis on Linguistic Properties, Model Architecture, and Input and Learning Objective of cross-lingual ability of Multilingual BERT","abstract":" ","authors":["Karthikeyan K","Zihan Wang","Stephen Mayhew","Dan Roth"],"iclr_id":"HJeT3yrtDr","keywords":["nlp"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Cross-Lingual Ability of Multilingual BERT: An Empirical Study"},"forum":"HJeT3yrtDr","id":"HJeT3yrtDr"},{"content":{"TLDR":"We predict the generalization error and specify the model which attains it across model/data scales.","abstract":" ","authors":["Jonathan S. Rosenfeld","Amir Rosenfeld","Yonatan Belinkov","Nir Shavit"],"iclr_id":"ryenvpEKDr","keywords":["generalization","generalization error","nlp","scalability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Constructive Prediction of the Generalization Error Across Scales"},"forum":"ryenvpEKDr","id":"ryenvpEKDr"},{"content":{"TLDR":"We introduce a method for unsupervised disentangled model selection for VAE-based disentangled representation learning approaches.","abstract":" ","authors":["Sunny Duan","Loic Matthey","Andre Saraiva","Nick Watters","Chris Burgess","Alexander Lerchner","Irina Higgins"],"iclr_id":"SyxL2TNtvr","keywords":["disentanglement","fairness","reinforcement learning","representation learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Unsupervised Model Selection for Variational Disentangled Representation Learning"},"forum":"SyxL2TNtvr","id":"SyxL2TNtvr"},{"content":{"TLDR":"Instead of fine-tuning after pruning, rewind weights or learning rate schedule to their values earlier in training and retrain from there to achieve higher accuracy when pruning neural networks.","abstract":" ","authors":["Alex Renda","Jonathan Frankle","Michael Carbin"],"iclr_id":"S1gSj0NKvB","keywords":["compression","fine tuning","learning rate","pruning","sparsity"],"recs":[],"session":["Thursday: Network Architectures"],"session_links":[null],"session_times":[null],"title":"Comparing Rewinding and Fine-tuning in Neural Network Pruning"},"forum":"S1gSj0NKvB","id":"S1gSj0NKvB"},{"content":{"TLDR":"We provide for the first time a rigorous proof that orthogonal initialization speeds up convergence relative to Gaussian initialization, for deep linear networks.","abstract":" ","authors":["Wei Hu","Lechao Xiao","Jeffrey Pennington"],"iclr_id":"rkgqN1SYvr","keywords":["deep learning theory","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Provable Benefit of Orthogonal Initialization in Optimizing Deep Linear Networks"},"forum":"rkgqN1SYvr","id":"rkgqN1SYvr"},{"content":{"TLDR":"Accurate, Fast and Automated Kernel-Wise Neural Network Quantization with Mixed Precision using Hierarchical Deep Reinforcement Learning","abstract":" ","authors":["Qian Lou","Feng Guo","Minje Kim","Lantao Liu","Lei Jiang."],"iclr_id":"rygfnn4twS","keywords":["automl","cnn","quantization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"AutoQ: Automated Kernel-Wise Neural Network Quantization "},"forum":"rygfnn4twS","id":"rygfnn4twS"},{"content":{"TLDR":"Learning compositional Koopman operators for efficient system identification and model-based control.","abstract":" ","authors":["Yunzhu Li","Hao He","Jiajun Wu","Dina Katabi","Antonio Torralba"],"iclr_id":"H1ldzA4tPr","keywords":["compositionality","generalization","graph networks"],"recs":[],"session":["Wednesday: RL and Planning"],"session_links":[null],"session_times":[null],"title":"Learning Compositional Koopman Operators for Model-Based Control"},"forum":"H1ldzA4tPr","id":"H1ldzA4tPr"},{"content":{"TLDR":"Anomaly detection method that uses: openset techniques for better generalization, random-transformation classification for non-image data.","abstract":" ","authors":["Liron Bergman","Yedid Hoshen"],"iclr_id":"H1lK_lBtvS","keywords":["anomaly detection","generalization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Classification-Based Anomaly Detection for General Data"},"forum":"H1lK_lBtvS","id":"H1lK_lBtvS"},{"content":{"TLDR":"Sequence generation models are commonly refined with reinforcement learning over user-defined metrics. However, high gradient variance hinders the practical use of this method. To stabilize this method, we adapt to contextual generation of categorica...","abstract":" ","authors":["Xinjie Fan","Yizhe Zhang","Zhendong Wang","Mingyuan Zhou"],"iclr_id":"r1lOgyrKDS","keywords":["generation","policy gradient","program synthesis","reinforcement learning","uncertainty","variance reduction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Adaptive Correlated Monte Carlo for Contextual Categorical Sequence Generation"},"forum":"r1lOgyrKDS","id":"r1lOgyrKDS"},{"content":{"TLDR":"We present local ensembles, a method for detecting extrapolation in trained models, which approximates the variance of an ensemble using local-second order information.","abstract":" ","authors":["David Madras","James Atwood","Alexander D'Amour"],"iclr_id":"BJl6bANtwH","keywords":["active learning","ensembles","reliability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Detecting Extrapolation with Local Ensembles"},"forum":"BJl6bANtwH","id":"BJl6bANtwH"},{"content":{"TLDR":"Learning to reason about higher order logic formulas in the latent space.","abstract":" ","authors":["Dennis Lee","Christian Szegedy","Markus Rabe","Sarah Loos","Kshitij Bansal"],"iclr_id":"Ske31kBtPr","keywords":["graph networks","reasoning"],"recs":[],"session":["Wednesday: Symbols and Discovery"],"session_links":[null],"session_times":[null],"title":"Mathematical Reasoning in Latent Space"},"forum":"Ske31kBtPr","id":"Ske31kBtPr"},{"content":{"TLDR":"We use supervised learning (and in particular deep learning) to produce better space partitions for fast nearest neighbor search.","abstract":" ","authors":["Yihe Dong","Piotr Indyk","Ilya Razenshteyn","Tal Wagner"],"iclr_id":"rkenmREFDr","keywords":["locality sensitive hashing","quantization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning Space Partitions for Nearest Neighbor Search"},"forum":"rkenmREFDr","id":"rkenmREFDr"},{"content":{"TLDR":"A framework that conducts online refinement of pseudo labels with a novel soft softmax-triplet loss for unsupervised domain adaptation on person re-identification.","abstract":" ","authors":["Yixiao Ge","Dapeng Chen","Hongsheng Li"],"iclr_id":"rJlnOhVYPS","keywords":["clustering","domain adaptation","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Mutual Mean-Teaching: Pseudo Label Refinery for Unsupervised Domain Adaptation on Person Re-identification"},"forum":"rJlnOhVYPS","id":"rJlnOhVYPS"},{"content":{"TLDR":"We show that regularizing the $\\ell_1$-norm of gradients improves robustness to post-training quantization in neural networks.","abstract":" ","authors":["Milad Alizadeh","Arash Behboodi","Mart van Baalen","Christos Louizos","Tijmen Blankevoort","Max Welling"],"iclr_id":"ryxK0JBtPr","keywords":["imagenet","memory","perturbation","quantization","regularization","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Gradient $\\ell_1$ Regularization for Quantization Robustness"},"forum":"ryxK0JBtPr","id":"ryxK0JBtPr"},{"content":{"TLDR":"We utilize attention to restrict equivariant neural networks to the set or co-occurring transformations in data. ","abstract":" ","authors":["David W. Romero","Mark Hoogendoorn"],"iclr_id":"r1g6ogrtDr","keywords":["attention","equivariance"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Co-Attentive Equivariant Neural Networks: Focusing Equivariance On Transformations Co-Occurring in Data"},"forum":"r1g6ogrtDr","id":"r1g6ogrtDr"},{"content":{"TLDR":"Learning to Estimate Point-Prediction Uncertainty and Correct Output in Neural Networks","abstract":" ","authors":["Xin Qiu","Elliot Meyerson","Risto Miikkulainen"],"iclr_id":"rkxNh1Stvr","keywords":["gaussian process","regression","uncertainty","uncertainty estimation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Quantifying Point-Prediction Uncertainty in Neural Networks via Residual Estimation with an I/O Kernel"},"forum":"rkxNh1Stvr","id":"rkxNh1Stvr"},{"content":{"TLDR":"We apply reinforcement learning to score-based causal discovery and achieve promising results on both synthetic and real datasets","abstract":" ","authors":["Shengyu Zhu","Ignavier Ng","Zhitang Chen"],"iclr_id":"S1g2skStPB","keywords":["optimization","reinforcement learning","structure learning"],"recs":[],"session":["Wednesday: Actions and Counterfactuals"],"session_links":[null],"session_times":[null],"title":"Causal Discovery with Reinforcement Learning"},"forum":"S1g2skStPB","id":"S1g2skStPB"},{"content":{"TLDR":"Overlearning means that a model trained for a seemingly simple objective implicitly learns to recognize attributes and concepts that are (1) not part of the learning objective, and (2) sensitive from a privacy or bias perspective.","abstract":" ","authors":["Congzheng Song","Vitaly Shmatikov"],"iclr_id":"SJeNz04tDS","keywords":["privacy","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Overlearning Reveals Sensitive Attributes"},"forum":"SJeNz04tDS","id":"SJeNz04tDS"},{"content":{"TLDR":"Wide neural networks can escape the NTK regime and couple with quadratic models, with provably nice optimization landscape and better generalization.","abstract":" ","authors":["Yu Bai","Jason D. Lee"],"iclr_id":"rkllGyBFPH","keywords":["deep learning theory","generalization","neural tangent kernel","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Beyond Linearization: On Quadratic and Higher-Order Approximation of Wide Neural Networks"},"forum":"rkllGyBFPH","id":"rkllGyBFPH"},{"content":{"TLDR":"We represent a computer program using a set of simpler programs and use this representation to improve program synthesis techniques.","abstract":" ","authors":["Augustus Odena","Charles Sutton"],"iclr_id":"rylHspEKPr","keywords":["program synthesis"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning to Represent Programs with Property Signatures"},"forum":"rylHspEKPr","id":"rylHspEKPr"},{"content":{"TLDR":"Gradient clipping doesn't endow robustness to label noise, but a simple loss-based variant does.","abstract":" ","authors":["Aditya Krishna Menon","Ankit Singh Rawat","Sashank J. Reddi","Sanjiv Kumar"],"iclr_id":"rklB76EKPr","keywords":["gradient descent","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Can gradient clipping mitigate label noise?"},"forum":"rklB76EKPr","id":"rklB76EKPr"},{"content":{"TLDR":"Deep neural networks often have millions of parameters. This can hinder their deployment to low-end devices, not only due to high memory requirements but also because of increased latency at inference. We propose a novel model compression method that...","abstract":" ","authors":["Tao Lin","Sebastian U. Stich","Luis Barba","Daniil Dmitriev","Martin Jaggi"],"iclr_id":"SJem8lSFwB","keywords":["compression","imagenet","memory","model compression","pruning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Dynamic Model Pruning with Feedback"},"forum":"SJem8lSFwB","id":"SJem8lSFwB"},{"content":{"TLDR":"We proposed a specific back-propagation method via proper spectral sub-gradient to integrate determinantal point process to deep learning framework.","abstract":" ","authors":["Tianshu Yu","Yikang Li","Baoxin Li"],"iclr_id":"rkeIq2VYPr","keywords":["computer vision","optimization","stability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep Learning of Determinantal Point Processes via Proper Spectral Sub-gradient"},"forum":"rkeIq2VYPr","id":"rkeIq2VYPr"},{"content":{"TLDR":"VariBAD opens a path to tractable approximate Bayes-optimal exploration for deep RL using ideas from meta-learning, Bayesian RL, and approximate variational inference.","abstract":" ","authors":["Luisa Zintgraf","Kyriacos Shiarlis","Maximilian Igl","Sebastian Schulze","Yarin Gal","Katja Hofmann","Shimon Whiteson"],"iclr_id":"Hkl9JlBYvr","keywords":["meta learning","reinforcement learning","uncertainty"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"VariBAD: A Very Good Method for Bayes-Adaptive Deep RL via Meta-Learning"},"forum":"Hkl9JlBYvr","id":"Hkl9JlBYvr"},{"content":{"TLDR":"This paper proposes a differentiable linear quadratic Model Predictive Control (MPC) framework for safe imitation learning. The infinite-horizon cost is enforced using a terminal cost function obtained from the discrete-time algebraic Riccati equatio...","abstract":" ","authors":["Sebastian East","Marco Gallieri","Jonathan Masci","Jan Koutnik","Mark Cannon"],"iclr_id":"ryxC6kSYPr","keywords":["imitation learning","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Infinite-Horizon Differentiable Model Predictive Control"},"forum":"ryxC6kSYPr","id":"ryxC6kSYPr"},{"content":{"TLDR":"This work addresses continual learning for non-stationary data, using Bayesian neural networks and memory-based online variational Bayes.","abstract":" ","authors":["Richard Kurle","Botond Cseke","Alexej Klushyn","Patrick van der Smagt","Stephan G\u00fcnnemann"],"iclr_id":"SJlsFpVtDB","keywords":["bayesian neural networks","continual learning","episodic memory","lifelong learning","memory","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Continual Learning with Bayesian Neural Networks for Non-Stationary Data"},"forum":"SJlsFpVtDB","id":"SJlsFpVtDB"},{"content":{"TLDR":"We provide a new framework for MAML in the ES/blackbox setting, and show that it allows deterministic and linear policies, better exploration, and non-differentiable adaptation operators.","abstract":" ","authors":["Xingyou Song","Wenbo Gao","Yuxiang Yang","Krzysztof Choromanski","Aldo Pacchiano","Yunhao Tang"],"iclr_id":"S1exA2NtDB","keywords":["maml","meta learning","perturbation","policy gradient","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"ES-MAML: Simple Hessian-Free Meta Learning"},"forum":"S1exA2NtDB","id":"S1exA2NtDB"},{"content":{"TLDR":"This paper shows how to train binary networks to within a few percent points (~3-5%) of the full precision counterpart. We first show how to build a strong baseline, which already achieves state-of-the-art accuracy, by combining recently proposed adv...","abstract":" ","authors":["Brais Martinez","Jing Yang","Adrian Bulat","Georgios Tzimiropoulos"],"iclr_id":"BJg4NgBKvH","keywords":["attention","imagenet","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Training binary neural networks with real-to-binary convolutions"},"forum":"BJg4NgBKvH","id":"BJg4NgBKvH"},{"content":{"TLDR":"A new recurrent neural network architecture for detecting pairwise Granger causality between nonlinearly interacting time series. ","abstract":" ","authors":["Saurabh Khanna","Vincent Y. F. Tan"],"iclr_id":"SyxV9ANFDH","keywords":["attention","causal inference","causality","overfitting","rnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Economy Statistical Recurrent Units For Inferring Nonlinear Granger Causality"},"forum":"SyxV9ANFDH","id":"SyxV9ANFDH"},{"content":{"TLDR":"\"A novel framework for overcoming catastrophic forgetting by preserving the distribution of the network's output at an arbitrary layer.\"","abstract":" ","authors":["Soheil Kolouri","Nicholas A. Ketz","Andrea Soltoggio","Praveen K. Pilly"],"iclr_id":"BJge3TNKwH","keywords":["capacity","catastrophic forgetting","incremental learning","memory","unsupervised"],"recs":[],"session":["Thursday: Continual Learning and Few Shot Learning"],"session_links":[null],"session_times":[null],"title":"Sliced Cramer Synaptic Consolidation for Preserving Deeply Learned Representations"},"forum":"BJge3TNKwH","id":"BJge3TNKwH"},{"content":{"TLDR":"Outputs of modern NLP APIs on nonsensical text provide strong signals about model internals, allowing adversaries to steal the APIs.","abstract":" ","authors":["Kalpesh Krishna","Gaurav Singh Tomar","Ankur P. Parikh","Nicolas Papernot","Mohit Iyyer"],"iclr_id":"Byl5NREFDr","keywords":["nlp","question answering","security","transfer learning","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Thieves on Sesame Street! Model Extraction of BERT-based APIs"},"forum":"Byl5NREFDr","id":"Byl5NREFDr"},{"content":{"TLDR":"The field of deep learning is commonly concerned with optimizing predictive models using large pre-acquired datasets of densely sampled datapoints or signals. In this work, we demonstrate that the deep learning paradigm can be extended to incorporate...","abstract":" ","authors":["Iris A.M. Huijben","Bastiaan S. Veeling","Ruud J.G. van Sloun"],"iclr_id":"SJeq9JBFvH","keywords":["optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep probabilistic subsampling for task-adaptive compressed sensing"},"forum":"SJeq9JBFvH","id":"SJeq9JBFvH"},{"content":{"TLDR":"ML algorithms or models, especially deep neural networks (DNNs), have shown significant promise in several areas. However, recently researchers have demonstrated that ML algorithms, especially DNNs, are vulnerable to adversarial examples (slightly pe...","abstract":" ","authors":["Uyeong Jang","Susmit Jha","Somesh Jha"],"iclr_id":"r1lF_CEYwS","keywords":["adversarial","adversarial attacks","generative models","robust learning","security"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On the Need for Topology-Aware Generative Models for Manifold-Based Defenses"},"forum":"r1lF_CEYwS","id":"r1lF_CEYwS"},{"content":{"TLDR":"We propose a hypothesis for why gradient descent generalizes based on how per-example gradients interact with each other.","abstract":" ","authors":["Satrajit Chatterjee"],"iclr_id":"ryeFY0EFwS","keywords":["generalization","gradient descent","optimization","overfitting"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Coherent Gradients: An Approach to Understanding Generalization in Gradient Descent-based Optimization"},"forum":"ryeFY0EFwS","id":"ryeFY0EFwS"},{"content":{"TLDR":"We propose a simple change to existing neural network structures for better defending against gradient-based adversarial attacks, using the k-winners-take-all activation function.","abstract":" ","authors":["Chang Xiao","Peilin Zhong","Changxi Zheng"],"iclr_id":"Skgvy64tvr","keywords":["adversarial","adversarial attacks","adversarial defense","distributed","robustness"],"recs":[],"session":["Monday: Robustness and Verification"],"session_links":[null],"session_times":[null],"title":"Enhancing Adversarial Defense by k-Winners-Take-All"},"forum":"Skgvy64tvr","id":"Skgvy64tvr"},{"content":{"TLDR":"State-of-the-art models are now trained with billions of parameters, reaching hardware limits in terms of memory consumption. This has created a recent demand for memory-efficient optimizers. To this end, we investigate the limits and performance tra...","abstract":" ","authors":["Xinyi Chen","Naman Agarwal","Elad Hazan","Cyril Zhang","Yi Zhang"],"iclr_id":"SklKcRNYDH","keywords":["memory","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Extreme Tensoring for Low-Memory Preconditioning "},"forum":"SklKcRNYDH","id":"SklKcRNYDH"},{"content":{"TLDR":"A memory architecture that support inferential reasoning.","abstract":" ","authors":["Andrea Banino","Adri\u00e0 Puigdom\u00e8nech Badia","Raphael K\u00f6ster","Martin J. Chadwick","Vinicius Zambaldi","Demis Hassabis","Caswell Barry","Matthew Botvinick","Dharshan Kumaran","Charles Blundell"],"iclr_id":"rJxlc0EtDr","keywords":["capacity","distributed","memory","memory augmented neural networks","reasoning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"MEMO: A Deep Network for Flexible Combination of Episodic Memories"},"forum":"rJxlc0EtDr","id":"rJxlc0EtDr"},{"content":{"TLDR":"Convolutional Neural Networks (CNNs) have emerged as highly successful tools for image generation, recovery, and restoration. A major contributing factor to this success is that convolutional networks impose strong prior assumptions about natural ima...","abstract":" ","authors":["Reinhard Heckel and Mahdi Soltanolkotabi"],"iclr_id":"HJeqhA4YDS","keywords":["cnn","denoising","generation","gradient descent","image generation","regularization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Denoising and Regularization via Exploiting the Structural Bias of Convolutional Generators"},"forum":"HJeqhA4YDS","id":"HJeqhA4YDS"},{"content":{"TLDR":"We propose an inductive matrix completion model without using side information. By factorizing the (rating) matrix into the product of low-dimensional latent embeddings of rows (users) and columns (items), a majority of existing matrix completion met...","abstract":" ","authors":["Muhan Zhang","Yixin Chen"],"iclr_id":"ByxxgCEYDS","keywords":["graph networks","transfer learning"],"recs":[],"session":["Thursday: Graphs and Representations"],"session_links":[null],"session_times":[null],"title":"Inductive Matrix Completion Based on Graph Neural Networks"},"forum":"ByxxgCEYDS","id":"ByxxgCEYDS"},{"content":{"TLDR":"A novel meta-RL method that infers latent subtask structure","abstract":" ","authors":["Sungryull Sohn","Hyunjae Woo","Jongwook Choi","Honglak Lee"],"iclr_id":"HkgsWxrtPB","keywords":["meta reinforcement learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Meta Reinforcement Learning with Autonomous Inference of Subtask Dependencies"},"forum":"HkgsWxrtPB","id":"HkgsWxrtPB"},{"content":{"TLDR":"This work proves the non-acceleration of Nesterov SGD with any hyper-parameters, and proposes new algorithm which provably accelerates SGD in the over-parameterized setting.","abstract":" ","authors":["Chaoyue Liu","Mikhail Belkin"],"iclr_id":"r1gixp4FPH","keywords":["acceleration","cnn","gradient descent","momentum"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Accelerating SGD with momentum for over-parameterized learning"},"forum":"r1gixp4FPH","id":"r1gixp4FPH"},{"content":{"TLDR":"We isolate one factor of RL generalization by analyzing the case when the agent only overfits to the observations. We show that architectural implicit regularizations occur in this regime.","abstract":" ","authors":["Xingyou Song","Yiding Jiang","Stephen Tu","Yilun Du","Behnam Neyshabur"],"iclr_id":"HJli2hNKDH","keywords":["generalization","overfitting","regularization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Observational Overfitting in Reinforcement Learning"},"forum":"HJli2hNKDH","id":"HJli2hNKDH"},{"content":{"TLDR":"A variant of Adam for strongly convex functions","abstract":" ","authors":["Guanghui Wang","Shiyin Lu","Quan Cheng","Wei-wei Tu","Lijun Zhang"],"iclr_id":"rye5YaEtPr","keywords":["adam","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"SAdam: A Variant of Adam for Strongly Convex Functions"},"forum":"rye5YaEtPr","id":"rye5YaEtPr"},{"content":{"TLDR":"While generative adversarial networks (GANs) have revolutionized machine learning, a number of open questions remain to fully understand them and exploit their power. One of these questions is how to efficiently achieve proper diversity and sampling ...","abstract":" ","authors":["Ze Wang","Xiuyuan Cheng","Guillermo Sapiro","Qiang Qiu"],"iclr_id":"S1lSapVtwS","keywords":["adversarial","gan","generation","image generation","perturbation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Stochastic Conditional Generative Networks with Basis Decomposition"},"forum":"S1lSapVtwS","id":"S1lSapVtwS"},{"content":{"TLDR":"A new methodology for novelty detection by utilizing hidden space activation values obtained from a deep autoencoder.","abstract":" ","authors":["Ki Hyun Kim","Sangwoo Shim","Yongsub Lim","Jongseob Jeon","Jeongwoo Choi","Byungchan Kim","Andre S. Yoon"],"iclr_id":"HkgeGeBYDB","keywords":["anomaly detection","autoencoder","novelty detection","outlier detection","semi supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"RaPP: Novelty Detection with Reconstruction along Projection Pathway"},"forum":"HkgeGeBYDB","id":"HkgeGeBYDB"},{"content":{"TLDR":"We introduce the 2-simplicial Transformer and show that this architecture is a useful inductive bias for logical reasoning in the context of deep reinforcement learning.","abstract":" ","authors":["James Clift","Dmitry Doryn","Daniel Murfet","James Wallbridge"],"iclr_id":"rkecJ6VFvr","keywords":["attention","inductive bias","logic","logical reasoning","reasoning","reinforcement learning","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Logic and the 2-Simplicial Transformer"},"forum":"rkecJ6VFvr","id":"rkecJ6VFvr"},{"content":{"TLDR":"We consider the challenge of estimating treatment effects from observational data; and point out that, in general, only some factors based on the observed covariates X contribute to selection of the treatment T, and only some to determining the outco...","abstract":" ","authors":["Negar Hassanpour","Russell Greiner"],"iclr_id":"HkxBJT4YvB","keywords":["regression"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning Disentangled Representations for CounterFactual Regression"},"forum":"HkxBJT4YvB","id":"HkxBJT4YvB"},{"content":{"TLDR":"We decompose the discriminator in a GAN in a principled way so that each component can be independently trained on different parts of the input. The resulting \"FactorGAN\" can be used for semi-supervised learning and in missing data scenarios.","abstract":" ","authors":["Daniel Stoller","Sebastian Ewert","Simon Dixon"],"iclr_id":"Hye1RJHKwB","keywords":["adversarial","gan","generation","image generation","missing data","semi supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Training Generative Adversarial Networks from Incomplete Observations using Factorised Discriminators"},"forum":"Hye1RJHKwB","id":"Hye1RJHKwB"},{"content":{"TLDR":"The first principled weight initialization method for hypernetworks","abstract":" ","authors":["Oscar Chang","Lampros Flokas","Hod Lipson"],"iclr_id":"H1lma24tPB","keywords":["hypernetworks","meta learning","optimization"],"recs":[],"session":["Thursday: Optimisation II"],"session_links":[null],"session_times":[null],"title":"Principled Weight Initialization for Hypernetworks"},"forum":"H1lma24tPB","id":"H1lma24tPB"},{"content":{"TLDR":"A method for learning quantization configuration for low precision networks that achieves state of the art performance for quantized networks.","abstract":" ","authors":["Steven K. Esser","Jeffrey L. McKinstry","Deepika Bablani","Rathinakumar Appuswamy","Dharmendra S. Modha"],"iclr_id":"rkgO66VKDS","keywords":["imagenet","quantization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"LEARNED STEP SIZE QUANTIZATION"},"forum":"rkgO66VKDS","id":"rkgO66VKDS"},{"content":{"TLDR":"We formulate a probabilistic latent sequence model to tackle unsupervised text style transfer, and show its effectiveness across a suite of unsupervised text style transfer tasks. ","abstract":" ","authors":["Junxian He","Xinyi Wang","Graham Neubig","Taylor Berg-Kirkpatrick"],"iclr_id":"HJlA0C4tPS","keywords":["adversarial","generative models","machine translation","nlp","unsupervised","variational inference"],"recs":[],"session":["Tuesday: Probabilistic Approaches"],"session_links":[null],"session_times":[null],"title":"A Probabilistic Formulation of Unsupervised Text Style Transfer"},"forum":"HJlA0C4tPS","id":"HJlA0C4tPS"},{"content":{"TLDR":"While generative adversarial networks (GAN) have been widely adopted in various topics, in this paper we generalize the standard GAN to a new perspective by treating realness as a random variable that can be estimated from multiple angles. In this ge...","abstract":" ","authors":["Yuanbo Xiangli","Yubin Deng","Bo Dai","Chen Change Loy","Dahua Lin"],"iclr_id":"B1lPaCNtPB","keywords":["adversarial","gan","generalization"],"recs":[],"session":["Monday: Generative models"],"session_links":[null],"session_times":[null],"title":"Real or Not Real, that is the Question"},"forum":"B1lPaCNtPB","id":"B1lPaCNtPB"},{"content":{"TLDR":"Machine learning algorithms have been increasingly deployed in critical automated decision-making systems that directly affect human lives. When these algorithms are solely trained to minimize the training/test error, they could suffer from systemati...","abstract":" ","authors":["Sina Baharlouei","Maher Nouiehed","Ahmad Beirami","Meisam Razaviyayn"],"iclr_id":"HkgsUJrtDB","keywords":["adversarial","clustering","fairness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"R\u00e9nyi Fair Inference"},"forum":"HkgsUJrtDB","id":"HkgsUJrtDB"},{"content":{"TLDR":"We provide a rigorous comparison of different Graph Neural Networks for graph classification.","abstract":" ","authors":["Federico Errica","Marco Podda","Davide Bacciu","Alessio Micheli"],"iclr_id":"HygDF6NFPB","keywords":["attention","graph networks","representation learning","reproducibility"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Fair Comparison of Graph Neural Networks for Graph Classification"},"forum":"HygDF6NFPB","id":"HygDF6NFPB"},{"content":{"TLDR":"We learn the arrow of time for MDPs and use it to measure reachability, detect side-effects and obtain a curiosity reward signal. ","abstract":" ","authors":["Nasim Rahaman","Steffen Wolf","Anirudh Goyal","Roman Remme","Yoshua Bengio"],"iclr_id":"rylJkpEtwS","keywords":["reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning the Arrow of Time for Problems in Reinforcement Learning"},"forum":"rylJkpEtwS","id":"rylJkpEtwS"},{"content":{"TLDR":"We explore the role of multiplicative interaction as a unifying framework to describe a range of classical and modern neural network architectural motifs, such as gating, attention layers, hypernetworks, and dynamic convolutions amongst others.","abstract":" ","authors":["Siddhant M. Jayakumar","Wojciech M. Czarnecki","Jacob Menick","Jonathan Schwarz","Jack Rae","Simon Osindero","Yee Whye Teh","Tim Harley","Razvan Pascanu"],"iclr_id":"rylnK6VtDH","keywords":["attention","hypernetworks","inductive bias"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Multiplicative Interactions and Where to Find Them"},"forum":"rylnK6VtDH","id":"rylnK6VtDH"},{"content":{"TLDR":"In this paper, we propose Imitative Models to combine the benefits of IL and goal-directed planning: probabilistic predictive models of desirable behavior able to plan interpretable expert-like trajectories to achieve specified goals.","abstract":" ","authors":["Nicholas Rhinehart","Rowan McAllister","Sergey Levine"],"iclr_id":"Skl4mRNYDr","keywords":["autonomous driving","imitation learning","planning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep Imitative Models for Flexible Inference, Planning, and Control"},"forum":"Skl4mRNYDr","id":"Skl4mRNYDr"},{"content":{"TLDR":"We augment model-free policy learning with a sequence-level surrogate reward functions and count-based visitation bonus and demonstrate effectiveness in the large batch, low-round regime seen in designing DNA and protein sequences.","abstract":" ","authors":["Christof Angermueller","David Dohan","David Belanger","Ramya Deshpande","Kevin Murphy","Lucy Colwell"],"iclr_id":"HklxbgBKvr","keywords":["capacity","model based reinforcement learning","optimization","reinforcement learning","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Model-based reinforcement learning for biological sequence design"},"forum":"HklxbgBKvr","id":"HklxbgBKvr"},{"content":{"TLDR":"We propose SWAP, a distributed algorithm for large-batch training of neural networks.","abstract":" ","authors":["Vipul Gupta","Santiago Akle Serrano","Dennis DeCoste"],"iclr_id":"rygFWAEFwS","keywords":["computer vision","distributed","generalization","imagenet"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Stochastic Weight Averaging in Parallel: Large-Batch Training That Generalizes Well"},"forum":"rygFWAEFwS","id":"rygFWAEFwS"},{"content":{"TLDR":"signal propagation theory applied to continuous surrogates of binary nets;  counter intuitive initialisation; reparameterisation trick not helpful","abstract":" ","authors":["George Stamatescu","Federica Gerace","Carlo Lucibello","Ian Fuss","Langford White"],"iclr_id":"rylmoxrFDH","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Critical initialisation in continuous approximations of binary neural networks"},"forum":"rylmoxrFDH","id":"rylmoxrFDH"},{"content":{"TLDR":" Neural ordinary differential equations (ODEs) have been attracting increasing attention in various research domains recently.  There have been some works studying optimization issues and approximation capabilities of neural ODEs, but their robustnes...","abstract":" ","authors":["Hanshu YAN","Jiawei DU","Vincent TAN","Jiashi FENG"],"iclr_id":"B1e9Y2NYvS","keywords":["adversarial","attention","cnn","optimization","perturbation","robustness"],"recs":[],"session":["Monday: Robustness and Verification"],"session_links":[null],"session_times":[null],"title":"On Robustness of Neural Ordinary Differential Equations"},"forum":"B1e9Y2NYvS","id":"B1e9Y2NYvS"},{"content":{"TLDR":"Higher momentum parameter $\\beta$ helps for escaping saddle points faster","abstract":" ","authors":["Jun-Kun Wang","Chi-Heng Lin","Jacob Abernethy"],"iclr_id":"rkeNfp4tPr","keywords":["gradient descent","momentum","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Escaping Saddle Points Faster with Stochastic Momentum"},"forum":"rkeNfp4tPr","id":"rkeNfp4tPr"},{"content":{"TLDR":"We show that perturbations based-on intermediate feature distributions yield more transferable adversarial examples and allow for analysis of the affects of adversarial perturbations on intermediate representations.","abstract":" ","authors":["Nathan Inkawhich","Kevin Liang","Lawrence Carin","Yiran Chen"],"iclr_id":"rJxAo2VYwr","keywords":["adversarial","adversarial attacks","imagenet","interpretability","perturbation","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Transferable Perturbations of Deep Feature Distributions"},"forum":"rJxAo2VYwr","id":"rJxAo2VYwr"},{"content":{"TLDR":"We present KG-A2C, a reinforcement learning agent that builds a dynamic knowledge graph while exploring and generates natural language using a template-based action space - outperforming all current agents on a wide set of text-based games.","abstract":" ","authors":["Prithviraj Ammanabrolu","Matthew Hausknecht"],"iclr_id":"B1x6w0EtwH","keywords":["generation","knowledge graphs","natural language generation","nlp","partial observability","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Graph Constrained Reinforcement Learning for Natural Language Action Spaces"},"forum":"B1x6w0EtwH","id":"B1x6w0EtwH"},{"content":{"TLDR":"Energy-based models trained on crystallized protein structures predict native side chain configurations and automatically discover molecular energy features.","abstract":" ","authors":["Yilun Du","Joshua Meier","Jerry Ma","Rob Fergus","Alexander Rives"],"iclr_id":"S1e_9xrFvS","keywords":["transformer"],"recs":[],"session":["Tuesday: Biology and ML"],"session_links":[null],"session_times":[null],"title":"Energy-based models for atomic-resolution protein conformations"},"forum":"S1e_9xrFvS","id":"S1e_9xrFvS"},{"content":{"TLDR":"Previous studies have found that an adversary attacker can often infer unintended input information from intermediate-layer features. We study the possibility of preventing such adversarial inference, yet without too much accuracy degradation. We pro...","abstract":" ","authors":["Liyao Xiang","Hao Zhang","Haotian Ma","Yifan Zhang","Jie Ren","Quanshi Zhang"],"iclr_id":"S1xFl64tDr","keywords":["adversarial","privacy"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Interpretable Complex-Valued Neural Networks for Privacy Protection"},"forum":"S1xFl64tDr","id":"S1xFl64tDr"},{"content":{"TLDR":"Policy gradient methods with actor-critic schemes demonstrate tremendous empirical successes, especially when the actors and critics are parameterized by neural networks. However, it remains less clear whether such \"neural\" policy gradient methods co...","abstract":" ","authors":["Lingxiao Wang","Qi Cai","Zhuoran Yang","Zhaoran Wang"],"iclr_id":"BJgQfkSYDS","keywords":["policy gradient"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Neural Policy Gradient Methods: Global Optimality and Rates of Convergence"},"forum":"BJgQfkSYDS","id":"BJgQfkSYDS"},{"content":{"TLDR":"The paper analyzes the effect of shot number on prototypical networks and proposes a robust method when the shot number differs from meta-training to meta-testing time.","abstract":" ","authors":["Tianshi Cao","Marc T Law","Sanja Fidler"],"iclr_id":"HkgB2TNYPS","keywords":["fewshot learning","meta learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Theoretical Analysis of the Number of Shots in Few-Shot Learning"},"forum":"HkgB2TNYPS","id":"HkgB2TNYPS"},{"content":{"TLDR":"Directional message passing incorporates spatial directional information to improve graph neural networks.","abstract":" ","authors":["Johannes Klicpera","Janek Gro\u00df","Stephan G\u00fcnnemann"],"iclr_id":"B1eWbxStPH","keywords":["equivariance","graph networks"],"recs":[],"session":["Tuesday: Probabilistic Approaches"],"session_links":[null],"session_times":[null],"title":"Directional Message Passing for Molecular Graphs"},"forum":"B1eWbxStPH","id":"B1eWbxStPH"},{"content":{"TLDR":"We propose HiPPO, a stable Hierarchical Reinforcement Learning algorithm that can train several levels of the hierarchy simultaneously, giving good performance both in skill discovery and adaptation.","abstract":" ","authors":["Alexander Li","Carlos Florensa","Ignasi Clavera","Pieter Abbeel"],"iclr_id":"ByeWogStDS","keywords":["hierarchical reinforcement learning","optimization","policy gradient","reinforcement learning","robustness","skill discovery","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Sub-policy Adaptation for Hierarchical Reinforcement Learning"},"forum":"ByeWogStDS","id":"ByeWogStDS"},{"content":{"TLDR":"A novel graph pooling method considering relationships between different nodes via conditional random fields.","abstract":" ","authors":["Hao Yuan","Shuiwang Ji"],"iclr_id":"BJxg_hVtwH","keywords":["clustering","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"StructPool: Structured Graph Pooling via Conditional Random Fields"},"forum":"BJxg_hVtwH","id":"BJxg_hVtwH"},{"content":{"TLDR":"We provide theoretical support to uncertainty estimates for deep learning obtained fitting random priors.","abstract":" ","authors":["Kamil Ciosek","Vincent Fortuin","Ryota Tomioka","Katja Hofmann","Richard Turner"],"iclr_id":"BJlahxHYDS","keywords":["bayesian inference","calibration","computer vision","ensembles","gaussian process","uncertainty","uncertainty estimation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Conservative Uncertainty Estimation By Fitting  Prior Networks"},"forum":"BJlahxHYDS","id":"BJlahxHYDS"},{"content":{"TLDR":"Our method infers constraints on task execution by leveraging the principle of maximum entropy to quantify how demonstrations differ from expected, un-constrained behavior.","abstract":" ","authors":["Dexter R.R. Scobee","S. Shankar Sastry"],"iclr_id":"BJliakStvH","keywords":["imitation learning","inverse reinforcement learning","reinforcement learning"],"recs":[],"session":["Tuesday: RL and Estimation"],"session_links":[null],"session_times":[null],"title":"Maximum Likelihood Constraint Inference for Inverse Reinforcement Learning"},"forum":"BJliakStvH","id":"BJliakStvH"},{"content":{"TLDR":"A neural simulation of Universal Turing Machine","abstract":" ","authors":["Hung Le","Truyen Tran","Svetha Venkatesh"],"iclr_id":"rkxxA24FDr","keywords":["fewshot learning","memory","memory augmented neural networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Neural Stored-program Memory"},"forum":"rkxxA24FDr","id":"rkxxA24FDr"},{"content":{"TLDR":"A method that trains large capacity neural networks with significantly improved accuracy and lower dynamic computational cost","abstract":" ","authors":["Babak Ehteshami Bejnordi","Tijmen Blankevoort","Max Welling"],"iclr_id":"Bke89JBtvB","keywords":["capacity","image classification","imagenet","semantic segmentation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Batch-shaping for learning conditional channel gated networks"},"forum":"Bke89JBtvB","id":"Bke89JBtvB"},{"content":{"TLDR":"We propose a metric for comparing data distributions based on their geometry while not relying on any positional information.","abstract":" ","authors":["Anton Tsitsulin","Marina Munkhoeva","Davide Mottin","Panagiotis Karras","Alex Bronstein","Ivan Oseledets","Emmanuel Mueller"],"iclr_id":"HyebplHYwB","keywords":["generative models"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"The Shape of Data: Intrinsic Distance for Data Distributions"},"forum":"HyebplHYwB","id":"HyebplHYwB"},{"content":{"TLDR":"We isolate the environmental and training factors that contribute to emergent systematic generalization in a situated language-learning agent","abstract":" ","authors":["Felix Hill","Andrew Lampinen","Rosalia Schneider","Stephen Clark","Matthew Botvinick","James L. McClelland","Adam Santoro"],"iclr_id":"SklGryBtwr","keywords":["compositionality","generalization","nlp"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Environmental drivers of systematicity and generalization in a situated agent"},"forum":"SklGryBtwr","id":"SklGryBtwr"},{"content":{"TLDR":"Detecting objects such as cars and pedestrians in 3D plays an indispensable role in autonomous driving. Existing approaches largely rely on expensive LiDAR sensors for accurate depth information. While recently pseudo-LiDAR has been introduced as a p...","abstract":" ","authors":["Yurong You","Yan Wang","Wei-Lun Chao","Divyansh Garg","Geoff Pleiss","Bharath Hariharan","Mark Campbell","Kilian Q. Weinberger"],"iclr_id":"BJedHRVtPB","keywords":["autonomous driving"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Pseudo-LiDAR++: Accurate Depth for 3D Object Detection in Autonomous Driving"},"forum":"BJedHRVtPB","id":"BJedHRVtPB"},{"content":{"TLDR":"We thoroughly investigate neural network learning dynamics over the early phase of training, finding that these changes are crucial and difficult to approximate, though extended pretraining can recover them.","abstract":" ","authors":["Jonathan Frankle","David J. Schwab","Ari S. Morcos"],"iclr_id":"Hkl1iRNFwS","keywords":["gradient descent","lottery tickets"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"The Early Phase of Neural Network Training"},"forum":"Hkl1iRNFwS","id":"Hkl1iRNFwS"},{"content":{"TLDR":"We consider large-scale retrieval problems such as question answering retrieval and present a comprehensive study of how different sentence level pre-training improving the BERT-style token-level pre-training for two-tower Transformer models.","abstract":" ","authors":["Wei-Cheng Chang","Felix X. Yu","Yin-Wen Chang","Yiming Yang","Sanjiv Kumar"],"iclr_id":"rkg-mA4FDr","keywords":["attention","nlp","representation learning","transformer","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Pre-training Tasks for Embedding-based Large-scale Retrieval"},"forum":"rkg-mA4FDr","id":"rkg-mA4FDr"},{"content":{"TLDR":"A central question of representation learning asks under which conditions it is possible to reconstruct the true latent variables of an arbitrarily complex generative process. Recent breakthrough work by Khemakhem et al. (2019) on nonlinear ICA has a...","abstract":" ","authors":["Peter Sorrenson","Carsten Rother","Ullrich K\u00f6the"],"iclr_id":"rygeHgSFDH","keywords":["disentanglement","nonlinear ica","representation learning"],"recs":[],"session":["Tuesday: Representation Learning"],"session_links":[null],"session_times":[null],"title":"Disentanglement by Nonlinear ICA with General Incompressible-flow Networks (GIN)"},"forum":"rygeHgSFDH","id":"rygeHgSFDH"},{"content":{"TLDR":"This work enforces Hamiltonian dynamics with control to learn system models from embedded position and velocity data, and exploits this physically-consistent dynamics to synthesize model-based control via energy shaping.","abstract":" ","authors":["Yaofeng Desmond Zhong","Biswadip Dey","Amit Chakraborty"],"iclr_id":"ryxmb1rKDS","keywords":["generalization","inductive bias","momentum"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Symplectic ODE-Net: Learning Hamiltonian Dynamics with Control"},"forum":"ryxmb1rKDS","id":"ryxmb1rKDS"},{"content":{"TLDR":"This paper proposes the use of spectral element methods for fast and accurate training of Neural Ordinary Differential Equations for system identification.","abstract":" ","authors":["Alessio Quaglino","Marco Gallieri","Jonathan Masci","Jan Koutn\u00edk"],"iclr_id":"Sye0XkBKvS","keywords":["generalization","memory","optimization","rnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"SNODE: Spectral Discretization of Neural ODEs for System Identification"},"forum":"Sye0XkBKvS","id":"Sye0XkBKvS"},{"content":{"TLDR":"Proposing a new counterfactual-based methodology to evaluate the hypotheses generated from saliency maps about deep RL agent behavior. ","abstract":" ","authors":["Akanksha Atrey","Kaleigh Clary","David Jensen"],"iclr_id":"rkl3m1BFDB","keywords":["interpretability","reasoning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Exploratory Not Explanatory: Counterfactual Analysis of Saliency Maps for Deep Reinforcement Learning"},"forum":"rkl3m1BFDB","id":"rkl3m1BFDB"},{"content":{"TLDR":"Class-conditional generative models hold promise to overcome the shortcomings of their discriminative counterparts. They are a natural choice to solve discriminative tasks in a robust manner as they jointly optimize for predictive performance and acc...","abstract":" ","authors":["Ethan Fetaya","Joern-Henrik Jacobsen","Will Grathwohl","Richard Zemel"],"iclr_id":"r1lPleBFvH","keywords":["adversarial","generative models","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Understanding the Limitations of Conditional Generative Models"},"forum":"r1lPleBFvH","id":"r1lPleBFvH"},{"content":{"TLDR":"This paper investigates a population-based training regime based on game-theoretic principles called Policy-Spaced Response Oracles (PSRO). PSRO is general in the sense that it (1) encompasses well-known algorithms such as fictitious play and double ...","abstract":" ","authors":["Paul Muller","Shayegan Omidshafiei","Mark Rowland","Karl Tuyls","Julien Perolat","Siqi Liu","Daniel Hennes","Luke Marris","Marc Lanctot","Edward Hughes","Zhe Wang","Guy Lever","Nicolas Heess","Thore Graepel","Remi Munos"],"iclr_id":"Bkl5kxrKDr","keywords":["game theory","multiagent"],"recs":[],"session":["Monday: Multiagent Systems"],"session_links":[null],"session_times":[null],"title":"A Generalized Training Approach for Multiagent Learning"},"forum":"Bkl5kxrKDr","id":"Bkl5kxrKDr"},{"content":{"TLDR":"We propose a method for computing adversarially robust representations in an entirely unsupervised way.","abstract":" ","authors":["Taylan Cemgil","Sumedh Ghaisas","Krishnamurthy (Dj) Dvijotham","Pushmeet Kohli"],"iclr_id":"H1gfFaEYDS","keywords":["adversarial","autoencoder","regularization","unsupervised","variational autoencoders","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Adversarially Robust Representations with Smooth Encoders"},"forum":"H1gfFaEYDS","id":"H1gfFaEYDS"},{"content":{"TLDR":"We propose a new variant of Q-learning algorithm called Maxmin Q-learning which provides a parameter-tuning mechanism to flexibly control bias.","abstract":" ","authors":["Qingfeng Lan","Yangchen Pan","Alona Fyshe","Martha White"],"iclr_id":"Bkg0u3Etwr","keywords":["generalization","reinforcement learning","variance reduction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Maxmin Q-learning: Controlling the Estimation Bias of Q-learning"},"forum":"Bkg0u3Etwr","id":"Bkg0u3Etwr"},{"content":{"TLDR":"We systematically examine why knowledge distillation is crucial to the training of non-autoregressive translation (NAT) models, and propose methods to further improve the distilled data to best match the capacity of an NAT model.","abstract":" ","authors":["Chunting Zhou","Jiatao Gu","Graham Neubig"],"iclr_id":"BygFVAEKDH","keywords":["autoregressive models","capacity","generation","knowledge distillation","machine translation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Understanding Knowledge Distillation in Non-autoregressive Machine Translation"},"forum":"BygFVAEKDH","id":"BygFVAEKDH"},{"content":{"TLDR":"We propose precision gating (PG), an end-to-end trainable dynamic dual-precision quantization technique for deep neural networks.","abstract":" ","authors":["Yichi Zhang","Ritchie Zhao","Weizhe Hua","Nayun Xu","G. Edward Suh","Zhiru Zhang"],"iclr_id":"SJgVU0EKwS","keywords":["imagenet","quantization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Precision Gating: Improving Neural Network Efficiency with Dynamic Dual-Precision Activations"},"forum":"SJgVU0EKwS","id":"SJgVU0EKwS"},{"content":{"TLDR":"Natural question generation (QG) aims to generate questions from a passage and an answer. Previous works on QG either (i) ignore the rich structure information hidden in text, (ii) solely rely on cross-entropy loss that leads to issues like exposure ...","abstract":" ","authors":["Yu Chen","Lingfei Wu","Mohammed J. Zaki"],"iclr_id":"HygnDhEtvr","keywords":["generation","graph networks","nlp","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Reinforcement Learning Based Graph-to-Sequence Model for Natural Question Generation"},"forum":"HygnDhEtvr","id":"HygnDhEtvr"},{"content":{"TLDR":"Through multi-agent competition, the simple objective of hide-and-seek, and standard reinforcement learning algorithms at scale, we find that agents create a self-supervised autocurriculum inducing multiple distinct rounds of emergent strategy, many ...","abstract":" ","authors":["Bowen Baker","Ingmar Kanitscheider","Todor Markov","Yi Wu","Glenn Powell","Bob McGrew","Igor Mordatch"],"iclr_id":"SkxpxJBKwS","keywords":["fine tuning","intrinsic motivation","reinforcement learning"],"recs":[],"session":["Monday: Multiagent Systems"],"session_links":[null],"session_times":[null],"title":"Emergent Tool Use From Multi-Agent Autocurricula"},"forum":"SkxpxJBKwS","id":"SkxpxJBKwS"},{"content":{"TLDR":"A continual learning framework which learns to automatically adapt its architecture based on a proposed variational inference algorithm. ","abstract":" ","authors":["Tameem Adel","Han Zhao","Richard E. Turner"],"iclr_id":"Hklso24Kwr","keywords":["catastrophic forgetting","continual learning","transfer learning","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Continual Learning with Adaptive Weights (CLAW)"},"forum":"Hklso24Kwr","id":"Hklso24Kwr"},{"content":{"TLDR":"A novel meta-learning model that adaptively balances the effect of the meta-learning and task-specific learning, and also class-specific learning within each task.","abstract":" ","authors":["Hae Beom Lee","Hayeon Lee","Donghyun Na","Saehoon Kim","Minseop Park","Eunho Yang","Sung Ju Hwang"],"iclr_id":"rkeZIJBYvr","keywords":["bayesian inference","fewshot learning","meta learning","variational inference"],"recs":[],"session":["Tuesday: Probabilistic Approaches"],"session_links":[null],"session_times":[null],"title":"Learning to Balance: Bayesian Meta-Learning for Imbalanced and Out-of-distribution Tasks"},"forum":"rkeZIJBYvr","id":"rkeZIJBYvr"},{"content":{"TLDR":"Translation into morphologically-rich languages challenges neural machine translation (NMT) models with extremely sparse vocabularies where atomic treatment of surface forms is unrealistic. This problem is typically addressed by either pre-processing...","abstract":" ","authors":["Duygu Ataman","Wilker Aziz","Alexandra Birch"],"iclr_id":"BJxSI1SKDH","keywords":["capacity","generalization","machine translation","nlp"],"recs":[],"session":["Thursday: Natural Language"],"session_links":[null],"session_times":[null],"title":"A Latent Morphology Model for Open-Vocabulary Neural Machine Translation"},"forum":"BJxSI1SKDH","id":"BJxSI1SKDH"},{"content":{"TLDR":"A scalable differentiable neural module that implements reasoning on symbolic KBs.","abstract":" ","authors":["William W. Cohen","Haitian Sun","R. Alex Hofer","Matthew Siegler"],"iclr_id":"BJlguT4YPr","keywords":["distributed","knowledge base completion","question answering","reasoning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Scalable Neural Methods for Reasoning With a Symbolic Knowledge   Base"},"forum":"BJlguT4YPr","id":"BJlguT4YPr"},{"content":{"TLDR":"We propose a new homotopy-based method to transfer \"optimality knowledge\" across different data distributions in order to speed up training of deep models.  ","abstract":" ","authors":["Matilde Gargiani","Andrea Zanelli","Quoc Tran Dinh","Moritz Diehl","Frank Hutter"],"iclr_id":"S1gEIerYwH","keywords":["fine tuning","gradient descent","optimization","regression","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Transferring Optimality Across Data Distributions via Homotopy Methods"},"forum":"S1gEIerYwH","id":"S1gEIerYwH"},{"content":{"TLDR":"A new partially policy-agnostic method for infinite-horizon off-policy policy evalution with multiple known or unknown behavior policies.","abstract":" ","authors":["Xinyun Chen","Lu Wang","Yizhe Hang","Heng Ge","Hongyuan Zha"],"iclr_id":"rkgU1gHtvr","keywords":["importance sampling","off policy policy evaluation","variance reduction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Infinite-horizon Off-Policy Policy Evaluation with Multiple Behavior Policies"},"forum":"rkgU1gHtvr","id":"rkgU1gHtvr"},{"content":{"TLDR":"We learn a space of motor primitives from unannotated robot demonstrations, and show these primitives are semantically meaningful and can be composed for new robot tasks.","abstract":" ","authors":["Tanmay Shankar","Shubham Tulsiani","Lerrel Pinto","Abhinav Gupta"],"iclr_id":"rkgHY0NYwr","keywords":["hierarchical reinforcement learning","imitation learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Discovering Motor Programs by Recomposing Demonstrations"},"forum":"rkgHY0NYwr","id":"rkgHY0NYwr"},{"content":{"TLDR":"We developed an effective parallel UCT algorithm that achieves linear speedup and suffers negligible performance loss.","abstract":" ","authors":["Anji Liu","Jianshu Chen","Mingze Yu","Yu Zhai","Xuewen Zhou","Ji Liu"],"iclr_id":"BJlQtJSKDB","keywords":["reinforcement learning"],"recs":[],"session":["Wednesday: RL and Planning"],"session_links":[null],"session_times":[null],"title":"Watch the Unobserved: A Simple Approach to Parallelizing Monte Carlo Tree Search"},"forum":"BJlQtJSKDB","id":"BJlQtJSKDB"},{"content":{"TLDR":"We propose Bayesian quantized networks, for which we learn a posterior distribution over their quantized parameters.","abstract":" ","authors":["Jiahao Su","Milan Cvitkovic","Furong Huang"],"iclr_id":"rylVHR4FPB","keywords":["bayesian neural networks","uncertainty"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Sampling-Free Learning of Bayesian Quantized Neural Networks"},"forum":"rylVHR4FPB","id":"rylVHR4FPB"},{"content":{"TLDR":"We highlight the problems with common metrics of in-domain uncertainty and perform a broad study of modern ensembling techniques.","abstract":" ","authors":["Arsenii Ashukha","Alexander Lyzhov","Dmitry Molchanov","Dmitry Vetrov"],"iclr_id":"BJxI5gHKDr","keywords":["ensemble learning","ensembles","uncertainty","uncertainty estimation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Pitfalls of In-Domain Uncertainty Estimation and Ensembling in Deep Learning"},"forum":"BJxI5gHKDr","id":"BJxI5gHKDr"},{"content":{"TLDR":"In the early phase of training of deep neural networks there exists a \"break-even point\" which determines properties of the entire optimization trajectory.","abstract":" ","authors":["Stanislaw Jastrzebski","Maciej Szymczak","Stanislav Fort","Devansh Arpit","Jacek Tabor","Kyunghyun Cho","Krzysztof Geras"],"iclr_id":"r1g87C4KwB","keywords":["batch normalization","generalization","gradient descent","learning rate","loss surface","optimization"],"recs":[],"session":["Thursday: Optimisation II"],"session_links":[null],"session_times":[null],"title":"The Break-Even Point on Optimization Trajectories of Deep Neural Networks"},"forum":"r1g87C4KwB","id":"r1g87C4KwB"},{"content":{"TLDR":"We characterize the space of functions realizable as a ReLU network with an unbounded number of units (infinite width), but where the Euclidean norm of the weights is bounded.","abstract":" ","authors":["Greg Ongie","Rebecca Willett","Daniel Soudry","Nathan Srebro"],"iclr_id":"H1lNPxHKDH","keywords":["inductive bias","regularization","relu networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Function Space View of Bounded Norm Infinite Width ReLU Nets: The Multivariate Case"},"forum":"H1lNPxHKDH","id":"H1lNPxHKDH"},{"content":{"TLDR":"We propose a novel normalization method to handle small batch size cases.","abstract":" ","authors":["Junjie Yan","Ruosi Wan","Xiangyu Zhang","Wei Zhang","Yichen Wei","Jian Sun"],"iclr_id":"SkgGjRVKDS","keywords":["batch normalization","computer vision","imagenet","memory"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Towards Stabilizing Batch Statistics in Backward Propagation of Batch Normalization"},"forum":"SkgGjRVKDS","id":"SkgGjRVKDS"},{"content":{"TLDR":"Learning overcomplete representations finds many applications in machine learning and data analytics. In the past decade, despite the empirical success of heuristic methods, theoretical understandings and explanations of these algorithms are still fa...","abstract":" ","authors":["Qing Qu","Yuexiang Zhai","Xiao Li","Yuqian Zhang","Zhihui Zhu"],"iclr_id":"rygixkHKDH","keywords":["dictionary learning","optimization","representation learning"],"recs":[],"session":["Wednesday: Optimisation I"],"session_links":[null],"session_times":[null],"title":"Geometric Analysis of Nonconvex Optimization Landscapes for Overcomplete Learning"},"forum":"rygixkHKDH","id":"rygixkHKDH"},{"content":{"TLDR":"By differentiating misclassified and correctly classified data, we propose a new misclassification aware defense that improves the state-of-the-art adversarial robustness.","abstract":" ","authors":["Yisen Wang","Difan Zou","Jinfeng Yi","James Bailey","Xingjun Ma","Quanquan Gu"],"iclr_id":"rklOg6EFwS","keywords":["adversarial","adversarial defense","optimization","perturbation","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Improving Adversarial Robustness Requires Revisiting Misclassified Examples"},"forum":"rklOg6EFwS","id":"rklOg6EFwS"},{"content":{"TLDR":"We present a novel method of compression of deep Convolutional Neural Networks (CNNs) by weight sharing through a new representation of convolutional filters.","abstract":" ","authors":["Yingzhen Yang","Jiahui Yu","Nebojsa Jojic","Jun Huan","Thomas S. Huang"],"iclr_id":"S1xtORNFwH","keywords":["acceleration","cnn","compression","computer vision","neural architecture search","quantization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"FSNet: Compression of Deep Convolutional Neural Networks by Filter Summary"},"forum":"S1xtORNFwH","id":"S1xtORNFwH"},{"content":{"TLDR":"We propose a reinforcement learning agent to solve hard exploration games by learning a range of directed exploratory policies. ","abstract":" ","authors":["Adri\u00e0 Puigdom\u00e8nech Badia","Pablo Sprechmann","Alex Vitvitskyi","Daniel Guo","Bilal Piot","Steven Kapturowski","Olivier Tieleman","Martin Arjovsky","Alexander Pritzel","Andrew Bolt","Charles Blundell"],"iclr_id":"Sye57xStvB","keywords":["distributed","episodic memory","intrinsic motivation","memory","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Never Give Up: Learning Directed Exploration Strategies"},"forum":"Sye57xStvB","id":"Sye57xStvB"},{"content":{"TLDR":"We prove that Transformer networks are universal approximators of sequence-to-sequence functions.","abstract":" ","authors":["Chulhee Yun","Srinadh Bhojanapalli","Ankit Singh Rawat","Sashank Reddi","Sanjiv Kumar"],"iclr_id":"ByxRM0Ntvr","keywords":["attention","equivariance","expressive power","permutation equivariance","self attention","transformer","universal approximation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Are Transformers universal approximators of sequence-to-sequence functions?"},"forum":"ByxRM0Ntvr","id":"ByxRM0Ntvr"},{"content":{"TLDR":"Defenses against adversarial attacks can be classified into certified and non-certified. Certifiable defenses make networks robust within a certain $\\ell_p$-bounded radius, so that it is impossible for the adversary to make adversarial examples in th...","abstract":" ","authors":["Amin Ghiasi","Ali Shafahi","Tom Goldstein"],"iclr_id":"HJxdTxHYvB","keywords":["adversarial","adversarial attacks","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"BREAKING  CERTIFIED  DEFENSES:  SEMANTIC  ADVERSARIAL  EXAMPLES  WITH  SPOOFED  ROBUSTNESS  CERTIFICATES"},"forum":"HJxdTxHYvB","id":"HJxdTxHYvB"},{"content":{"TLDR":"Deep associative memory models using arbitrary neural networks as a storage.","abstract":" ","authors":["Sergey Bartunov","Jack Rae","Simon Osindero","Timothy Lillicrap"],"iclr_id":"SyljQyBFDH","keywords":["associative memory","compression","memory","meta learning","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Meta-Learning Deep Energy-Based Memory Models"},"forum":"SyljQyBFDH","id":"SyljQyBFDH"},{"content":{"TLDR":"We propose a new dataset to investigate the entailment problem under semi-structured table as premise","abstract":" ","authors":["Wenhu Chen","Hongmin Wang","Jianshu Chen","Yunkai Zhang","Hong Wang","Shiyang Li","Xiyou Zhou","William Yang Wang"],"iclr_id":"rkeJRhNYDH","keywords":["fact verification","nlp","reasoning","tabular data","verification"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"TabFact: A Large-scale Dataset for Table-based Fact Verification"},"forum":"rkeJRhNYDH","id":"rkeJRhNYDH"},{"content":{"TLDR":"Sort in encoder and undo sorting in decoder to avoid responsibility problem in set auto-encoders","abstract":" ","authors":["Yan Zhang","Jonathon Hare","Adam Pr\u00fcgel-Bennett"],"iclr_id":"HJgBA2VYwH","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"FSPool: Learning Set Representations with Featurewise Sort Pooling"},"forum":"HJgBA2VYwH","id":"HJgBA2VYwH"},{"content":{"TLDR":"Learning a labeling policy with reinforcement learning to reduce labeling effort for the task of semantic segmentation","abstract":" ","authors":["Arantxa Casanova","Pedro O. Pinheiro","Negar Rostamzadeh","Christopher J. Pal"],"iclr_id":"SkgC6TNFvr","keywords":["active learning","reinforcement learning","semantic segmentation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Reinforced active learning for image segmentation"},"forum":"SkgC6TNFvr","id":"SkgC6TNFvr"},{"content":{"TLDR":"We develop and analyze a new derivative free optimization algorithm with momentum and importance sampling with applications to continuous control.","abstract":" ","authors":["Eduard Gorbunov","Adel Bibi","Ozan Sener","El Houcine Bergou","Peter Richtarik"],"iclr_id":"HylAoJSKvH","keywords":["continuous control","derivative free optimization","importance sampling","momentum","optimization","policy gradient"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Stochastic Derivative Free Optimization Method with Momentum"},"forum":"HylAoJSKvH","id":"HylAoJSKvH"},{"content":{"TLDR":"We present a real-time segmentation model automatically discovered by a multi-scale NAS framework, achieving 30% faster than state-of-the-art models.","abstract":" ","authors":["Wuyang Chen","Xinyu Gong","Xianming Liu","Qian Zhang","Yuan Li","Zhangyang Wang"],"iclr_id":"BJgqQ6NYvB","keywords":["neural architecture search","regularization","semantic segmentation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"FasterSeg: Searching for Faster Real-time Semantic Segmentation"},"forum":"BJgqQ6NYvB","id":"BJgqQ6NYvB"},{"content":{"TLDR":"We exploit the global linearity of the mixup-trained models in inference to break the locality of the adversarial perturbations.","abstract":" ","authors":["Tianyu Pang","Kun Xu","Jun Zhu"],"iclr_id":"ByxtC2VtPB","keywords":["adversarial","adversarial attacks","generalization","perturbation","robustness","trustworthy machine learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Mixup Inference: Better Exploiting Mixup to Defend Adversarial Attacks"},"forum":"ByxtC2VtPB","id":"ByxtC2VtPB"},{"content":{"TLDR":"We propose a meta path planning algorithm which exploits a novel attention-based neural module that can learn generalizable structures from prior experiences to drastically reduce the sample requirement for solving new path planning problems.","abstract":" ","authors":["Binghong Chen","Bo Dai","Qinjie Lin","Guo Ye","Han Liu","Le Song"],"iclr_id":"rJgJDAVKvB","keywords":["meta learning","planning","reinforcement learning","representation learning","sample efficiency"],"recs":[],"session":["Wednesday: RL and Planning"],"session_links":[null],"session_times":[null],"title":"Learning to Plan in High Dimensions via Neural Exploration-Exploitation Trees"},"forum":"rJgJDAVKvB","id":"rJgJDAVKvB"},{"content":{"TLDR":"We proposed a novel GAN framework to generate unseen data.","abstract":" ","authors":["Yi Lin Sung","Sung-Hsien Hsieh","Soo-Chang Pei","Chun-Shien Lu"],"iclr_id":"rygjmpVFvB","keywords":["adversarial","gan","generation","novelty detection","semi supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Difference-Seeking Generative Adversarial Network--Unseen Sample Generation"},"forum":"rygjmpVFvB","id":"rygjmpVFvB"},{"content":{"TLDR":"VL-BERT is a simple yet powerful pre-trainable generic representation for visual-linguistic tasks. It is pre-trained on the massive-scale caption dataset and text-only corpus, and can be finetuned for varies down-stream visual-linguistic tasks.","abstract":" ","authors":["Weijie Su","Xizhou Zhu","Yue Cao","Bin Li","Lewei Lu","Furu Wei","Jifeng Dai"],"iclr_id":"SygXPaEYvH","keywords":["pre training","question answering","reasoning","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"VL-BERT: Pre-training of Generic Visual-Linguistic Representations"},"forum":"SygXPaEYvH","id":"SygXPaEYvH"},{"content":{"TLDR":"Graph regularization forces spectral embedding to focus on the largest clusters, making the representation less sensitive to noise. ","abstract":" ","authors":["Nathan De Lara","Thomas Bonald"],"iclr_id":"H1l_0JBYwS","keywords":["clustering","regularization"],"recs":[],"session":["Tuesday: Feature Discovery for Structured Data"],"session_links":[null],"session_times":[null],"title":"Spectral  Embedding of Regularized Block Models"},"forum":"H1l_0JBYwS","id":"H1l_0JBYwS"},{"content":{"TLDR":"We use RL to automatically learn branching heuristic within a state of the art QBF solver, on industrial problems.","abstract":" ","authors":["Gil Lederman","Markus Rabe","Sanjit Seshia","Edward A. Lee"],"iclr_id":"BJluxREKDB","keywords":["graph networks","logic","logical reasoning","reasoning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning Heuristics for Quantified Boolean Formulas through Reinforcement Learning"},"forum":"BJluxREKDB","id":"BJluxREKDB"},{"content":{"TLDR":"A text encoder trained to distinguish real input tokens from plausible fakes efficiently learns effective language representations.","abstract":" ","authors":["Kevin Clark","Minh-Thang Luong","Quoc V. Le","Christopher D. Manning"],"iclr_id":"r1xMH1BtvB","keywords":["language modeling","nlp","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators"},"forum":"r1xMH1BtvB","id":"r1xMH1BtvB"},{"content":{"TLDR":"This paper aims to analyze knowledge consistency between pre-trained deep neural networks. We propose a generic definition for knowledge consistency between neural networks at different fuzziness levels. A task-agnostic method is designed to disentan...","abstract":" ","authors":["Ruofan Liang","Tianlin Li","Longfei Li","Jing Wang","Quanshi Zhang"],"iclr_id":"BJeS62EtwH","keywords":["cnn","compression","interpretability","knowledge distillation","network compression"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Knowledge Consistency between Neural Networks and Beyond"},"forum":"BJeS62EtwH","id":"BJeS62EtwH"},{"content":{"TLDR":"Meta-learning methods used for vision, directly applied to NLP, perform worse than nearest neighbors on new classes; we can do better with distributional signatures.","abstract":" ","authors":["Yujia Bao","Menghua Wu","Shiyu Chang","Regina Barzilay"],"iclr_id":"H1emfT4twB","keywords":["attention","computer vision","fewshot learning","meta learning","text classification"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Few-shot Text Classification with Distributional Signatures"},"forum":"H1emfT4twB","id":"H1emfT4twB"},{"content":{"TLDR":"A new operation called Harmonic Convolution makes deep network model audio priors without training.","abstract":" ","authors":["Zhoutong Zhang","Yunyun Wang","Chuang Gan","Jiajun Wu","Joshua B. Tenenbaum","Antonio Torralba","William T. Freeman"],"iclr_id":"rygjHxrYDB","keywords":["audio","cnn","generalization","generation","inductive bias","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep Audio Priors Emerge From Harmonic Convolutional Networks"},"forum":"rygjHxrYDB","id":"rygjHxrYDB"},{"content":{"TLDR":"alternative to gradient penalty","abstract":" ","authors":["D\u00e1vid Terj\u00e9k"],"iclr_id":"Bke_DertPB","keywords":["adversarial","gan","generative models","regularization","stability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Adversarial Lipschitz Regularization"},"forum":"Bke_DertPB","id":"Bke_DertPB"},{"content":{"TLDR":"A deep RL algorithm for solving POMDPs by auto-encoding the underlying states using a variational recurrent model","abstract":" ","authors":["Dongqi Han","Kenji Doya","Jun Tani"],"iclr_id":"r1lL4a4tDB","keywords":["continuous control","reinforcement learning","rnn","variational inference"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Variational Recurrent Models for Solving Partially Observable Control Tasks"},"forum":"r1lL4a4tDB","id":"r1lL4a4tDB"},{"content":{"TLDR":"We propose Stable Rank Normalisation, a new regularisor based on recent generelization bounds and show how to optimize it with extensive experiments.","abstract":" ","authors":["Amartya Sanyal","Philip H. Torr","Puneet K. Dokania"],"iclr_id":"H1enKkrFDB","keywords":["generalization","regularization"],"recs":[],"session":["Monday: Generative models"],"session_links":[null],"session_times":[null],"title":"Stable Rank Normalization for Improved Generalization in Neural Networks and GANs"},"forum":"H1enKkrFDB","id":"H1enKkrFDB"},{"content":{"TLDR":"An efficient, unbiased approximation of the softmax loss function for extreme classification","abstract":" ","authors":["Robert Bamler","Stephan Mandt"],"iclr_id":"rJxe3xSYDS","keywords":["adversarial","regression"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Extreme Classification via Adversarial Softmax Approximation"},"forum":"rJxe3xSYDS","id":"rJxe3xSYDS"},{"content":{"TLDR":"We propose Additive Powers-of-Two~(APoT) quantization, an efficient non-uniform quantization scheme for the bell-shaped and long-tailed distribution of weights and activations in neural networks. By constraining all quantization levels as the sum of ...","abstract":" ","authors":["Yuhang Li","Xin Dong","Wei Wang"],"iclr_id":"BkgXT24tDS","keywords":["efficient inference","imagenet","quantization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Additive Powers-of-Two Quantization: An Efficient Non-uniform Discretization for Neural Networks"},"forum":"BkgXT24tDS","id":"BkgXT24tDS"},{"content":{"TLDR":"Identifiability, or recovery of the true latent representations from which the observed data originates, is de facto a fundamental goal of representation learning. Yet, most deep generative models do not address the question of identifiability, and t...","abstract":" ","authors":["Shen Li","Bryan Hooi","Gim Hee Lee"],"iclr_id":"SklOUpEYvB","keywords":["autoencoder","generative models","nonlinear ica","optimization","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Identifying through Flows for Recovering Latent Representations"},"forum":"SklOUpEYvB","id":"SklOUpEYvB"},{"content":{"TLDR":"Acquire states from high frequency region for search-control in Dyna.","abstract":" ","authors":["Yangchen Pan","Jincheng Mei","Amir-massoud Farahmand"],"iclr_id":"B1gskyStwr","keywords":["model based reinforcement learning","planning","reinforcement learning","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Frequency-based Search-control in Dyna"},"forum":"B1gskyStwr","id":"B1gskyStwr"},{"content":{"TLDR":"Actor-Critic method with function approximation finds the Nash equilibrium pairs in mean-field games with theoretical guarantee. ","abstract":" ","authors":["Zuyue Fu","Zhuoran Yang","Yongxin Chen","Zhaoran Wang"],"iclr_id":"H1lhqpEYPr","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Actor-Critic Provably Finds Nash Equilibria of Linear-Quadratic Mean-Field Games"},"forum":"H1lhqpEYPr","id":"H1lhqpEYPr"},{"content":{"TLDR":"Approximate inference using dynamic programming for Autoregressive models.","abstract":" ","authors":["Najam Zaidi","Trevor Cohn","Gholamreza Haffari"],"iclr_id":"HklOo0VFDH","keywords":["autoregressive models","generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Decoding As Dynamic Programming For Recurrent Autoregressive Models"},"forum":"HklOo0VFDH","id":"HklOo0VFDH"},{"content":{"TLDR":"Horizontal and top-down feedback connections are responsible for complementary perceptual grouping strategies in biological and recurrent vision systems.","abstract":" ","authors":["Junkyung Kim","Drew Linsley","Kalpit Thakkar","Thomas Serre"],"iclr_id":"HJxrVA4FDS","keywords":["recurrent feedback","visual cortex"],"recs":[],"session":["Tuesday: Biology and ML"],"session_links":[null],"session_times":[null],"title":"Disentangling neural mechanisms for perceptual grouping"},"forum":"HJxrVA4FDS","id":"HJxrVA4FDS"},{"content":{"TLDR":"We apply the informational bottleneck concept to attribution.","abstract":" ","authors":["Karl Schulz","Leon Sixt","Federico Tombari","Tim Landgraf"],"iclr_id":"S1xWh1rYwB","keywords":["information bottleneck","interpretability","information bottleneck","information theory"],"recs":[],"session":["Tuesday: Feature Discovery for Structured Data"],"session_links":[null],"session_times":[null],"title":"Restricting the Flow: Information Bottlenecks for Attribution"},"forum":"S1xWh1rYwB","id":"S1xWh1rYwB"},{"content":{"TLDR":"Recent theoretical work has guaranteed that overparameterized networks trained by gradient descent achieve arbitrarily low training error, and sometimes even low test error.\nThe required width, however, is always polynomial in at least one of the sam...","abstract":" ","authors":["Ziwei Ji","Matus Telgarsky"],"iclr_id":"HygegyrYwH","keywords":["gradient descent","neural tangent kernel","relu networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Polylogarithmic width suffices for gradient descent to achieve arbitrarily small test error with shallow ReLU networks"},"forum":"HygegyrYwH","id":"HygegyrYwH"},{"content":{"TLDR":"A model to control the generation of images with GAN and beta-VAE with regard to scale and position of the objects","abstract":" ","authors":["Antoine Plumerault","Herv\u00e9 Le Borgne","C\u00e9line Hudelot"],"iclr_id":"H1laeJrKDB","keywords":["computer vision","gan","generative models","interpretability","nlp"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Controlling generative models with continuous factors of variations"},"forum":"H1laeJrKDB","id":"H1laeJrKDB"},{"content":{"TLDR":"We show that Energy-Based models when trained on the residual of an auto-regressive language model can be used effectively and efficiently to generate text. ","abstract":" ","authors":["Yuntian Deng","Anton Bakhtin","Myle Ott","Arthur Szlam","Marc'Aurelio Ranzato"],"iclr_id":"B1l4SgHKDH","keywords":["energy based models","generation","importance sampling","language modeling","machine translation","nlp","text generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Residual Energy-Based Models for Text Generation"},"forum":"B1l4SgHKDH","id":"B1l4SgHKDH"},{"content":{"TLDR":"Current language generation systems either aim for high likelihood and devolve into generic repetition or miscalibrate their stochasticity\u2014we provide evidence of both and propose a solution: Nucleus Sampling.","abstract":" ","authors":["Ari Holtzman","Jan Buys","Li Du","Maxwell Forbes","Yejin Choi"],"iclr_id":"rygGQyrFvH","keywords":["generation","language modeling","natural language generation","nlp","text generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"The Curious Case of Neural Text Degeneration"},"forum":"rygGQyrFvH","id":"rygGQyrFvH"},{"content":{"TLDR":"Long-range transformer using a compressive memory, achieves sota in wikitext-103 and enwik8 LM benchmarks, release a new book-level LM benchmark PG-19.","abstract":" ","authors":["Jack W. Rae","Anna Potapenko","Siddhant M. Jayakumar","Chloe Hillier","Timothy P. Lillicrap"],"iclr_id":"SylKikSYDH","keywords":["compression","language modeling","memory","nlp","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Compressive Transformers for Long-Range Sequence Modelling"},"forum":"SylKikSYDH","id":"SylKikSYDH"},{"content":{"TLDR":"While deep neural networks have achieved impressive performance on a range of NLP tasks, these data-hungry models heavily rely on labeled data, which restricts their applications in scenarios where data annotation is expensive. Natural language (NL) ...","abstract":" ","authors":["Ziqi Wang","Yujia Qin","Wenxuan Zhou","Jun Yan","Qinyuan Ye","Leonardo Neves","Zhiyuan Liu","Xiang Ren"],"iclr_id":"rJlUt0EYwS","keywords":["generalization","nlp","question answering","text classification"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning from Explanations with Neural Execution Tree"},"forum":"rJlUt0EYwS","id":"rJlUt0EYwS"},{"content":{"TLDR":"We propose almost everywhere differentiable and scale invariant regularizers for DNN pruning, which can lead to supremum sparsity through standard SGD training.","abstract":" ","authors":["Huanrui Yang","Wei Wen","Hai Li"],"iclr_id":"rylBK34FDS","keywords":["compression","gradient descent","model compression","optimization","pruning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"DeepHoyer: Learning Sparser Neural Network with Differentiable Scale-Invariant Sparsity Measures"},"forum":"rylBK34FDS","id":"rylBK34FDS"},{"content":{"TLDR":"We give some generalization error bounds of noisy gradient methods such as SGLD, Langevin dynamics, noisy momentum and so forth.","abstract":" ","authors":["Jian Li","Xuanyuan Luo","Mingda Qiao"],"iclr_id":"SkxxtgHKPS","keywords":["acceleration","attention","deep learning theory","generalization","gradient descent","momentum","regularization","stability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On Generalization Error Bounds of Noisy Gradient Methods for Non-Convex Learning"},"forum":"SkxxtgHKPS","id":"SkxxtgHKPS"},{"content":{"TLDR":"Task agnostic pre-training can shape RNN's attractor landscape, and form diverse inductive bias for different navigation tasks   ","abstract":" ","authors":["Tie XU","Omri Barak"],"iclr_id":"Byx4NkrtDS","keywords":["inductive bias","navigation","pre training","reinforcement learning","rnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Implementing Inductive bias for different navigation tasks through diverse RNN attrractors"},"forum":"Byx4NkrtDS","id":"Byx4NkrtDS"},{"content":{"TLDR":"We propose a new large-scale diverse environment for few-shot learning, and evaluate popular models' performance on it, revealing important research challenges.","abstract":" ","authors":["Eleni Triantafillou","Tyler Zhu","Vincent Dumoulin","Pascal Lamblin","Utku Evci","Kelvin Xu","Ross Goroshin","Carles Gelada","Kevin Swersky","Pierre-Antoine Manzagol","Hugo Larochelle"],"iclr_id":"rkgAGAVKPr","keywords":["fewshot learning","generalization","meta learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Meta-Dataset: A Dataset of Datasets for Learning to Learn from Few Examples"},"forum":"rkgAGAVKPr","id":"rkgAGAVKPr"},{"content":{"TLDR":"We derive a low-variance, unbiased gradient estimator for expectations over discrete random variables based on sampling without replacement","abstract":" ","authors":["Wouter Kool","Herke van Hoof","Max Welling"],"iclr_id":"rklEj2EFvB","keywords":["policy gradient","variational inference"],"recs":[],"session":["Tuesday: RL and Estimation"],"session_links":[null],"session_times":[null],"title":"Estimating Gradients for Discrete Random Variables by Sampling without Replacement"},"forum":"rklEj2EFvB","id":"rklEj2EFvB"},{"content":{"TLDR":"We propose MACER: a provable defense algorithm that trains robust models by maximizing the certified radius. It does not use adversarial training but performs better than all existing provable l2-defenses.","abstract":" ","authors":["Runtian Zhai","Chen Dan","Di He","Huan Zhang","Boqing Gong","Pradeep Ravikumar","Cho-Jui Hsieh","Liwei Wang"],"iclr_id":"rJx1Na4Fwr","keywords":["adversarial","adversarial defense","imagenet","randomized smoothing","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"MACER: Attack-free and Scalable Robust Training via Maximizing Certified Radius"},"forum":"rJx1Na4Fwr","id":"rJx1Na4Fwr"},{"content":{"TLDR":"Understanding causes and effects in mechanical systems is an essential component of reasoning in the physical world. This work poses a new problem of counterfactual learning of object mechanics from visual input.  We develop the CoPhy benchmark to as...","abstract":" ","authors":["Fabien Baradel","Natalia Neverova","Julien Mille","Greg Mori","Christian Wolf"],"iclr_id":"SkeyppEFvS","keywords":["capacity","reasoning","video prediction"],"recs":[],"session":["Wednesday: Actions and Counterfactuals"],"session_links":[null],"session_times":[null],"title":"CoPhy: Counterfactual Learning of Physical Dynamics"},"forum":"SkeyppEFvS","id":"SkeyppEFvS"},{"content":{"TLDR":"This work proposed a universal visual representation for neural machine translation (NMT) using retrieved images with similar topics to source sentence,  extending image applicability in NMT.","abstract":" ","authors":["Zhuosheng Zhang","Kehai Chen","Rui Wang","Masao Utiyama","Eiichiro Sumita","Zuchao Li","Hai Zhao"],"iclr_id":"Byl8hhNYPS","keywords":["attention","machine translation","nlp"],"recs":[],"session":["Thursday: Natural Language"],"session_links":[null],"session_times":[null],"title":"Neural Machine Translation with Universal Visual Representation"},"forum":"Byl8hhNYPS","id":"Byl8hhNYPS"},{"content":{"TLDR":"In natural language processing, it has been observed recently that generalization could be greatly improved by finetuning a large-scale language model pretrained on a large unlabeled corpus. Despite its recent success and wide adoption, finetuning a ...","abstract":" ","authors":["Cheolhyoung Lee","Kyunghyun Cho","Wanmo Kang"],"iclr_id":"HkgaETNtDB","keywords":["dropout","generalization","nlp","optimization","regularization","stability","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Mixout: Effective Regularization to Finetune Large-scale Pretrained Language Models"},"forum":"HkgaETNtDB","id":"HkgaETNtDB"},{"content":{"TLDR":"We search for multi-stream neural architectures with better connectivity and spatio-temporal interactions for video understanding.","abstract":" ","authors":["Michael S. Ryoo","AJ Piergiovanni","Mingxing Tan","Anelia Angelova"],"iclr_id":"SJgMK64Ywr","keywords":["neural architecture search","representation learning","video understanding"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"AssembleNet: Searching for Multi-Stream Neural Connectivity in Video Architectures"},"forum":"SJgMK64Ywr","id":"SJgMK64Ywr"},{"content":{"TLDR":"A flow-based autoregressive model for molecular graph generation. Reaching state-of-the-art results on molecule generation and properties optimization.","abstract":" ","authors":["Chence Shi","Minkai Xu","Zhaocheng Zhu","Weinan Zhang","Ming Zhang","Jian Tang"],"iclr_id":"S1esMkHYPr","keywords":["attention","autoregressive models","fine tuning","generation","generative models","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"GraphAF: a Flow-based Autoregressive Model for Molecular Graph Generation"},"forum":"S1esMkHYPr","id":"S1esMkHYPr"},{"content":{"TLDR":"We propose a novel optimization objective that encourages fairness in heterogeneous federated networks, and develop a scalable method to solve it.","abstract":" ","authors":["Tian Li","Maziar Sanjabi","Ahmad Beirami","Virginia Smith"],"iclr_id":"ByexElSYDr","keywords":["distributed","distributed optimization","fairness","federated learning","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Fair Resource Allocation in Federated Learning"},"forum":"ByexElSYDr","id":"ByexElSYDr"},{"content":{"TLDR":"We develop Simplified Action Decoder, a simple MARL algorithm that beats previous SOTA on Hanabi by a big margin across 2- to 5-player games.","abstract":" ","authors":["Hengyuan Hu","Jakob N Foerster"],"iclr_id":"B1xm3RVtwB","keywords":["adversarial","multi agent reinforcement learning","reinforcement learning"],"recs":[],"session":["Monday: Multiagent Systems"],"session_links":[null],"session_times":[null],"title":"Simplified Action Decoder for Deep Multi-Agent Reinforcement Learning"},"forum":"B1xm3RVtwB","id":"B1xm3RVtwB"},{"content":{"TLDR":"A hybrid training technique that combines ANN-SNN conversion and spike-based backpropagation to optimize training effort and inference latency.","abstract":" ","authors":["Nitin Rathi","Gopalakrishnan Srinivasan","Priyadarshini Panda","Kaushik Roy"],"iclr_id":"B1xSperKvH","keywords":["imagenet"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Enabling Deep Spiking Neural Networks with Hybrid Conversion and Spike Timing Dependent Backpropagation"},"forum":"B1xSperKvH","id":"B1xSperKvH"},{"content":{"TLDR":"Given a pre-trained model, we explored the per-sample gradients of the model parameters relative to a task-specific loss, and constructed a linear model that combines gradients of model parameters and the activation of the model.","abstract":" ","authors":["Fangzhou Mu","Yingyu Liang","Yin Li"],"iclr_id":"BkeoaeHKDS","keywords":["representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Gradients as Features for Deep Representation Learning"},"forum":"BkeoaeHKDS","id":"BkeoaeHKDS"},{"content":{"TLDR":"We introduce a class of generative models that reliably learn Hamiltonian dynamics from high-dimensional observations. The learnt Hamiltonian can be applied to sequence modeling or as a normalising flow.","abstract":" ","authors":["Peter Toth","Danilo J. Rezende","Andrew Jaegle","S\u00e9bastien Racani\u00e8re","Aleksandar Botev","Irina Higgins"],"iclr_id":"HJenn6VFvB","keywords":["generative models","reinforcement learning","rnn"],"recs":[],"session":["Monday: Generative models"],"session_links":[null],"session_times":[null],"title":"Hamiltonian Generative Networks"},"forum":"HJenn6VFvB","id":"HJenn6VFvB"},{"content":{"TLDR":"Use iterated learning framework to facilitate the dominance of high compositional language in multi-agent games.","abstract":" ","authors":["Yi Ren","Shangmin Guo","Matthieu Labeau","Shay B. Cohen","Simon Kirby"],"iclr_id":"HkePNpVKPB","keywords":["compositionality","generalization","multi agent","nlp"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Compositional languages emerge in a neural iterated learning model"},"forum":"HkePNpVKPB","id":"HkePNpVKPB"},{"content":{"TLDR":"Gradientless Descent is a provably efficient gradient-free algorithm that is monotone-invariant and fast for high-dimensional zero-th order optimization.","abstract":" ","authors":["Daniel Golovin","John Karro","Greg Kochanski","Chansoo Lee","Xingyou Song","Qiuyi Zhang"],"iclr_id":"Skep6TVYDB","keywords":["optimization","zeroth order optimization"],"recs":[],"session":["Monday: Optimisation Theory"],"session_links":[null],"session_times":[null],"title":"Gradientless Descent: High-Dimensional Zeroth-Order Optimization"},"forum":"Skep6TVYDB","id":"Skep6TVYDB"},{"content":{"TLDR":"A machine learning model that generalizes well should obtain low errors on unseen test examples. Thus, if we know how to optimally perturb training examples to account for test examples, we may achieve better generalization performance. However, obta...","abstract":" ","authors":["Hae Beom Lee","Taewook Nam","Eunho Yang","Sung Ju Hwang"],"iclr_id":"BJgd81SYwr","keywords":["dropout","generalization","information bottleneck","meta learning","perturbation","regularization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Meta Dropout: Learning to Perturb Latent Features for Generalization"},"forum":"BJgd81SYwr","id":"BJgd81SYwr"},{"content":{"TLDR":"We propose ranking policy gradient that learns the optimal rank of actions to maximize return. We propose a general off-policy learning framework with the properties of optimality preserving, variance reduction, and sample-efficiency.","abstract":" ","authors":["Kaixiang Lin","Jiayu Zhou"],"iclr_id":"rJld3hEYvS","keywords":["optimization","policy gradient","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Ranking Policy Gradient"},"forum":"rJld3hEYvS","id":"rJld3hEYvS"},{"content":{"TLDR":"We propose a new video understanding benchmark, with tasks that by-design require temporal reasoning to be solved, unlike most existing video datasets.","abstract":" ","authors":["Rohit Girdhar","Deva Ramanan"],"iclr_id":"HJgzt2VKPB","keywords":["computer vision","implicit bias","reasoning","video understanding"],"recs":[],"session":["Wednesday: Actions and Counterfactuals"],"session_links":[null],"session_times":[null],"title":"CATER: A diagnostic dataset for Compositional Actions & TEmporal Reasoning"},"forum":"HJgzt2VKPB","id":"HJgzt2VKPB"},{"content":{"TLDR":"We develop the foundations of a unifying theoretical framework connecting node embeddings and structural graph representations through invariant theory","abstract":" ","authors":["Balasubramaniam Srinivasan","Bruno Ribeiro"],"iclr_id":"SJxzFySKwH","keywords":["graph isomorphism","graph networks","relational learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On the Equivalence between Positional Node Embeddings and Structural Graph Representations"},"forum":"SJxzFySKwH","id":"SJxzFySKwH"},{"content":{"TLDR":"We study the structure of ridge regression in a high-dimensional asymptotic framework, and get insights about cross-validation and sketching.","abstract":" ","authors":["Sifan Liu","Edgar Dobriban"],"iclr_id":"HklRwaEKwB","keywords":["regression","regularization"],"recs":[],"session":["Wednesday: Theory"],"session_links":[null],"session_times":[null],"title":"Ridge Regression: Structure, Cross-Validation, and Sketching"},"forum":"HklRwaEKwB","id":"HklRwaEKwB"},{"content":{"TLDR":"We present an efficient approach to integrating numerical comparisons into differentiable rule learning in knowledge graphs","abstract":" ","authors":["Po-Wei Wang","Daria Stepanova","Csaba Domokos","J. Zico Kolter"],"iclr_id":"rJleKgrKwS","keywords":["knowledge graphs"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Differentiable learning of numerical rules in knowledge graphs"},"forum":"rJleKgrKwS","id":"rJleKgrKwS"},{"content":{"TLDR":"In Deep RL, order-invariant functions can be used in conjunction with standard memory modules to improve gradient decay and resilience to noise.","abstract":" ","authors":["Jacob Beck","Kamil Ciosek","Sam Devlin","Sebastian Tschiatschek","Cheng Zhang","Katja Hofmann"],"iclr_id":"Bkl7bREtDr","keywords":["memory","reinforcement learning","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"AMRL: Aggregated Memory For Reinforcement Learning"},"forum":"Bkl7bREtDr","id":"Bkl7bREtDr"},{"content":{"TLDR":"We introduce an efficient memory layer to jointly learn representations and coarsen the input graphs.","abstract":" ","authors":["Amir Hosein Khasahmadi","Kaveh Hassani","Parsa Moradi","Leo Lee","Quaid Morris"],"iclr_id":"r1laNeBYPB","keywords":["graph networks","memory","regression","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Memory-Based Graph Networks"},"forum":"r1laNeBYPB","id":"r1laNeBYPB"},{"content":{"TLDR":"We prove generalization bounds for recurrent neural networks based on matrix 1-norm and Fisher-Rao norm.","abstract":" ","authors":["Zhuozhuo Tu","Fengxiang He","Dacheng Tao"],"iclr_id":"rkgg6xBYDH","keywords":["deep learning theory","generalization","rnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Understanding Generalization in Recurrent Neural Networks"},"forum":"rkgg6xBYDH","id":"rkgg6xBYDH"},{"content":{"TLDR":"An analysis of the effects of compositionality and locality on representation learning for zero-shot learning.","abstract":" ","authors":["Tristan Sylvain","Linda Petrini","Devon Hjelm"],"iclr_id":"Hye_V0NKwr","keywords":["compositionality","generalization","imagenet","representation learning","zero shot learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Locality and Compositionality in Zero-Shot Learning"},"forum":"Hye_V0NKwr","id":"Hye_V0NKwr"},{"content":{"TLDR":"We distill an ensemble of models into a single model, capturing both the improved classification performance and information about the diversity of the ensemble, which is useful for uncertainty estimation.","abstract":" ","authors":["Andrey Malinin","Bruno Mlodozeniec","Mark Gales"],"iclr_id":"BygSP6Vtvr","keywords":["density estimation","ensembles","imagenet","knowledge distillation","memory","uncertainty","uncertainty estimation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Ensemble Distribution Distillation"},"forum":"BygSP6Vtvr","id":"BygSP6Vtvr"},{"content":{"TLDR":"As 3D scanning solutions become increasingly popular, several deep learning setups have been developed for the task of scan completion, i.e., plausibly filling in regions that were missed in the raw scans. These methods, however, largely rely on supe...","abstract":" ","authors":["Xuelin Chen","Baoquan Chen","Niloy J. Mitra"],"iclr_id":"HkgrZ0EYwB","keywords":["adversarial","gan"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Unpaired Point Cloud Completion on Real Scans using Adversarial Training"},"forum":"HkgrZ0EYwB","id":"HkgrZ0EYwB"},{"content":{"TLDR":"A NAS benchmark applicable to almost any NAS algorithms.","abstract":" ","authors":["Xuanyi Dong","Yi Yang"],"iclr_id":"HJxyZkBKDr","keywords":["automl","neural architecture search","regularization"],"recs":[],"session":["Thursday: Network Architectures"],"session_links":[null],"session_times":[null],"title":"NAS-Bench-201: Extending the Scope of Reproducible Neural Architecture Search"},"forum":"HJxyZkBKDr","id":"HJxyZkBKDr"},{"content":{"TLDR":"The success of MAML relies on feature reuse from the meta-initialization, which also yields a natural simplification of the algorithm, with the inner loop removed for the network body, as well as other insights on the head and body.","abstract":" ","authors":["Aniruddh Raghu","Maithra Raghu","Samy Bengio","Oriol Vinyals"],"iclr_id":"rkgMkCEtPB","keywords":["fewshot learning","meta learning","optimization","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Rapid Learning or Feature Reuse? Towards Understanding the Effectiveness of MAML"},"forum":"rkgMkCEtPB","id":"rkgMkCEtPB"},{"content":{"TLDR":"We propose a new regression framework for temporal phenomena having non-stationary time-lag dependencies.","abstract":" ","authors":["Mandar Chandorkar","Cyril Furtlehner","Bala Poduval","Enrico Camporeale","Michele Sebag"],"iclr_id":"SkxybANtDB","keywords":["regression","stability","time series"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Dynamic Time Lag Regression: Predicting What & When"},"forum":"SkxybANtDB","id":"SkxybANtDB"},{"content":{"TLDR":"This paper proposes a method of end-to-end multi-modal generation of human face from speech based on a self-supervised learning framework.","abstract":" ","authors":["Hyeong-Seok Choi","Changdae Park","Kyogu Lee"],"iclr_id":"H1guaREYPr","keywords":["generation","multi modal learning","self supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"From Inference to Generation: End-to-end Fully Self-supervised Generation of Human Face from Speech"},"forum":"H1guaREYPr","id":"H1guaREYPr"},{"content":{"TLDR":"We study the problem of regret minimization for distributed bandits learning, in which $M$ agents work collaboratively to minimize their total regret under the coordination of a central server. Our goal is to design communication protocols with near-...","abstract":" ","authors":["Yuanhao Wang","Jiachen Hu","Xiaoyu Chen","Liwei Wang"],"iclr_id":"SJxZnR4YvB","keywords":["distributed"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Distributed Bandit Learning: Near-Optimal Regret with Efficient Communication"},"forum":"SJxZnR4YvB","id":"SJxZnR4YvB"},{"content":{"TLDR":"We introduce a class of n-player games suited to gradient-based methods.","abstract":" ","authors":["David Balduzzi","Wojciech M. Czarnecki","Tom Anthony","Ian Gemp","Edward Hughes","Joel Leibo","Georgios Piliouras","Thore Graepel"],"iclr_id":"B1xMEerYvB","keywords":["adversarial","game theory","gradient descent","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Smooth markets: A basic mechanism for organizing gradient-based learners"},"forum":"B1xMEerYvB","id":"B1xMEerYvB"},{"content":{"TLDR":"We have presented LambdaNet, a neural architecture for type inference that combines the strength of explicit program analysis with graph neural networks.","abstract":" ","authors":["Jiayi Wei","Maruth Goyal","Greg Durrett","Isil Dillig"],"iclr_id":"Hkx6hANtwH","keywords":["graph networks","nlp"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"LambdaNet: Probabilistic Type Inference using Graph Neural Networks"},"forum":"Hkx6hANtwH","id":"Hkx6hANtwH"},{"content":{"TLDR":"We propose a modular framework that can accomplish tasks specified by programs and achieve zero-shot generalization to more complex tasks.","abstract":" ","authors":["Shao-Hua Sun","Te-Lin Wu","Joseph J. Lim"],"iclr_id":"BkxUvnEYDH","keywords":["generalization","learning to execute","nlp"],"recs":[],"session":["Wednesday: RL and Planning"],"session_links":[null],"session_times":[null],"title":"Program Guided Agent"},"forum":"BkxUvnEYDH","id":"BkxUvnEYDH"},{"content":{"TLDR":"A new state-of-the-art on Imagenet for mobile setting","abstract":" ","authors":["Jieru Mei","Yingwei Li","Xiaochen Lian","Xiaojie Jin","Linjie Yang","Alan Yuille","Jianchao Yang"],"iclr_id":"BylQSxHFwr","keywords":["image classification","imagenet","neural architecture search"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"AtomNAS: Fine-Grained End-to-End Neural Architecture Search"},"forum":"BylQSxHFwr","id":"BylQSxHFwr"},{"content":{"TLDR":"We create an unbiased estimator for the log probability of latent variable models, extending such models to a larger scope of applications.","abstract":" ","authors":["Yucen Luo","Alex Beatson","Mohammad Norouzi","Jun Zhu","David Duvenaud","Ryan P. Adams","Ricky T. Q. Chen"],"iclr_id":"SylkYeHtwr","keywords":[],"recs":[],"session":["Tuesday: Probabilistic Approaches"],"session_links":[null],"session_times":[null],"title":"SUMO: Unbiased Estimation of Log Marginal Probability for Latent Variable Models"},"forum":"SylkYeHtwr","id":"SylkYeHtwr"},{"content":{"TLDR":"We show that there is a hidden generative model inside of every classifier. We demonstrate how to train this model and show the many benefits of doing so.  ","abstract":" ","authors":["Will Grathwohl","Kuan-Chieh Wang","Joern-Henrik Jacobsen","David Duvenaud","Mohammad Norouzi","Kevin Swersky"],"iclr_id":"Hkxzx0NtDB","keywords":["adversarial","calibration","energy based models","generative models","outlier detection","robustness"],"recs":[],"session":["Monday: Generative models"],"session_links":[null],"session_times":[null],"title":"Your classifier is secretly an energy based model and you should treat it like one"},"forum":"Hkxzx0NtDB","id":"Hkxzx0NtDB"},{"content":{"TLDR":"Federated learning enables a large amount of edge computing devices to jointly learn a model without data sharing. As a leading algorithm in this setting, Federated Averaging (\\texttt{FedAvg}) runs Stochastic Gradient Descent (SGD) in parallel on a s...","abstract":" ","authors":["Xiang Li","Kaixuan Huang","Wenhao Yang","Shusen Wang","Zhihua Zhang"],"iclr_id":"HJxNAnVtDS","keywords":["federated learning","gradient descent","learning rate","optimization"],"recs":[],"session":["Monday: Optimisation Theory"],"session_links":[null],"session_times":[null],"title":"On the Convergence of FedAvg on Non-IID Data"},"forum":"HJxNAnVtDS","id":"HJxNAnVtDS"},{"content":{"TLDR":"A white noise analysis of modern deep neural networks is presented to unveil\ntheir biases at the whole network level or the single neuron level. Our analysis is\nbased on two popular and related methods in psychophysics and neurophysiology\nnamely clas...","abstract":" ","authors":["Ali Borji","Sikun Lin"],"iclr_id":"H1ebhnEYDH","keywords":["adversarial","adversarial defense","computational neuroscience","computer vision","imagenet","regression","visualization"],"recs":[],"session":["Monday: Robustness and Verification"],"session_links":[null],"session_times":[null],"title":"White Noise Analysis of Neural Networks"},"forum":"H1ebhnEYDH","id":"H1ebhnEYDH"},{"content":{"TLDR":"Deep  symbolic superoptimization refers to the task of applying deep learning methods to simplify symbolic expressions.   Existing approaches either perform supervised training on human-constructed datasets that defines equivalent expression pairs, o...","abstract":" ","authors":["Hui Shi","Yang Zhang","Xinyun Chen","Yuandong Tian","Jishen Zhao"],"iclr_id":"r1egIyBFPS","keywords":["attention","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep Symbolic Superoptimization Without Human Knowledge"},"forum":"r1egIyBFPS","id":"r1egIyBFPS"},{"content":{"TLDR":"We present Dreamer, an agent that learns long-horizon behaviors purely by latent imagination using analytic value gradients.","abstract":" ","authors":["Danijar Hafner","Timothy Lillicrap","Jimmy Ba","Mohammad Norouzi"],"iclr_id":"S1lOTC4tDS","keywords":["optimization","planning","reinforcement learning"],"recs":[],"session":["Wednesday: RL and Planning"],"session_links":[null],"session_times":[null],"title":"Dream to Control: Learning Behaviors by Latent Imagination"},"forum":"S1lOTC4tDS","id":"S1lOTC4tDS"},{"content":{"TLDR":"We construct a theoretical framework for weakly supervised disentanglement and conducted lots of experiments to back up the theory.","abstract":" ","authors":["Rui Shu","Yining Chen","Abhishek Kumar","Stefano Ermon","Ben Poole"],"iclr_id":"HJgSwyBKvr","keywords":["disentanglement","generative models","representation learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Weakly Supervised Disentanglement with Guarantees"},"forum":"HJgSwyBKvr","id":"HJgSwyBKvr"},{"content":{"TLDR":"We introduce R2D3, an agent that makes efficient use of demonstrations to solve hard exploration problems in partially observable environments with highly variable initial conditions.","abstract":" ","authors":["Caglar Gulcehre","Tom Le Paine","Bobak Shahriari","Misha Denil","Matt Hoffman","Hubert Soyer","Richard Tanburn","Steven Kapturowski","Neil Rabinowitz","Duncan Williams","Gabriel Barth-Maron","Ziyu Wang","Nando de Freitas","Worlds Team"],"iclr_id":"SygKyeHKDH","keywords":["imitation learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Making Efficient Use of Demonstrations to Solve Hard Exploration Problems"},"forum":"SygKyeHKDH","id":"SygKyeHKDH"},{"content":{"TLDR":"Contrastively-trained Structured World Models (C-SWMs) learn object-oriented state representations and a relational model of an environment from raw pixel input.","abstract":" ","authors":["Thomas Kipf","Elise van der Pol","Max Welling"],"iclr_id":"H1gax6VtDB","keywords":["graph networks","model based reinforcement learning","object discovery","reinforcement learning","relational learning","representation learning"],"recs":[],"session":["Thursday: Graphs and Representations"],"session_links":[null],"session_times":[null],"title":"Contrastive Learning of Structured World Models"},"forum":"H1gax6VtDB","id":"H1gax6VtDB"},{"content":{"TLDR":"Exploiting rich strucural details in graph-structued data via adaptive \"strucutral fingerprints''","abstract":" ","authors":["Kai Zhang","Yaokang Zhu","Jun Wang","Jie Zhang"],"iclr_id":"BJxWx0NYPr","keywords":["attention","graph networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Adaptive Structural Fingerprints for Graph Attention Networks"},"forum":"BJxWx0NYPr","id":"BJxWx0NYPr"},{"content":{"TLDR":"In multi-agent games, the complexity of the environment can grow exponentially as the number of agents increases, so it is particularly challenging to learn good policies when the agent population is large. In this paper, we introduce Evolutionary Po...","abstract":" ","authors":["Qian Long","Zihan Zhou","Abhinav Gupta","Fei Fang","Yi Wu\u2020","Xiaolong Wang\u2020"],"iclr_id":"SJxbHkrKDH","keywords":["curriculum learning","fine tuning","multi agent reinforcement learning","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Evolutionary Population Curriculum for Scaling Multi-Agent Reinforcement Learning"},"forum":"SJxbHkrKDH","id":"SJxbHkrKDH"},{"content":{"TLDR":"Applying resolution-preserving blocks is a common practice to maximize information preservation in video prediction, yet their high memory consumption greatly limits their application scenarios. We propose CrevNet, a Conditionally Reversible Network ...","abstract":" ","authors":["Wei Yu","Yichao Lu","Steve Easterbrook","Sanja Fidler"],"iclr_id":"B1eY_pVYvB","keywords":["autoencoder","memory","self supervised learning","transfer learning","video prediction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Efficient and Information-Preserving Future Frame Prediction and Beyond"},"forum":"B1eY_pVYvB","id":"B1eY_pVYvB"},{"content":{"TLDR":"We empirically study generalization measures over more than 2000 models, identify common pitfall in existing practice of studying generalization measures and provide some new bounds based on measures in our study.","abstract":" ","authors":["Yiding Jiang","Behnam Neyshabur","Hossein Mobahi","Dilip Krishnan","Samy Bengio"],"iclr_id":"SJgIPJBFvH","keywords":["generalization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Fantastic Generalization Measures and Where to Find Them"},"forum":"SJgIPJBFvH","id":"SJgIPJBFvH"},{"content":{"TLDR":"We propose an efficient, provable and data independent method for network compression via neural pruning using coresets of neurons -- a novel construction proposed in this paper.","abstract":" ","authors":["Ben Mussay","Margarita Osadchy","Vladimir Braverman","Samson Zhou","Dan Feldman"],"iclr_id":"H1gmHaEKwB","keywords":["adversarial","compression","memory","model compression","network compression","pruning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Data-Independent Neural Pruning via Coresets"},"forum":"H1gmHaEKwB","id":"H1gmHaEKwB"},{"content":{"TLDR":"MXGNet is a multilayer, multiplex graph based architecture which achieves good performance on various diagrammatic reasoning tasks.","abstract":" ","authors":["Duo Wang","Mateja Jamnik","Pietro Lio"],"iclr_id":"ByxQB1BKwH","keywords":["graph networks","reasoning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Abstract Diagrammatic Reasoning with Multiplex Graph Networks"},"forum":"ByxQB1BKwH","id":"ByxQB1BKwH"},{"content":{"TLDR":"A promising approach for teaching artificial agents to use natural language involves using human-in-the-loop training. However, recent work suggests that current machine learning methods are too data inefficient to be trained in this way from scratch...","abstract":" ","authors":["Ryan Lowe","Abhinav Gupta","Jakob Foerster","Douwe Kiela","Joelle Pineau"],"iclr_id":"rJxGLlBtwH","keywords":["nlp","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On the interaction between supervision and self-play in emergent communication"},"forum":"rJxGLlBtwH","id":"rJxGLlBtwH"},{"content":{"TLDR":"We formally characterize the initialization conditions for effective pruning at initialization and analyze the signal propagation properties of the resulting pruned networks which leads to a method to enhance their trainability and pruning results.","abstract":" ","authors":["Namhoon Lee","Thalaiyasingam Ajanthan","Stephen Gould","Philip H. S. Torr"],"iclr_id":"HJeTo2VFwH","keywords":["pruning","unsupervised"],"recs":[],"session":["Thursday: Network Architectures"],"session_links":[null],"session_times":[null],"title":"A Signal Propagation Perspective for Pruning Neural Networks at Initialization"},"forum":"HJeTo2VFwH","id":"HJeTo2VFwH"},{"content":{"TLDR":"We develop a new doubly robust estimator based on the infinite horizon density ratio and off policy value estimation.","abstract":" ","authors":["Ziyang Tang","Yihao Feng","Lihong Li","Dengyong Zhou","Qiang Liu"],"iclr_id":"S1glGANtDr","keywords":["importance sampling","reinforcement learning"],"recs":[],"session":["Monday: Reliable RL"],"session_links":[null],"session_times":[null],"title":"Doubly Robust Bias Reduction in Infinite Horizon Off-Policy Estimation"},"forum":"S1glGANtDr","id":"S1glGANtDr"},{"content":{"TLDR":"Neural architecture search (NAS) searches architectures automatically for given tasks, e.g., image classification and language modeling. Improving the search efficiency and effectiveness has attracted increasing attention in recent years. However, fe...","abstract":" ","authors":["Yao Shu","Wei Wang","Shaofeng Cai"],"iclr_id":"BJxH22EKPS","keywords":["attention","generalization","language modeling","loss landscape","neural architecture search","nlp","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Understanding Architectures Learnt by Cell-based Neural Architecture Search"},"forum":"BJxH22EKPS","id":"BJxH22EKPS"},{"content":{"TLDR":"Novel architecture for stereoscopic view synthesis at arbitrary camera shifts utilizing adaptive t-shaped kernels with adaptive dilations.","abstract":" ","authors":["Juan Luis Gonzalez Bello","Munchurl Kim"],"iclr_id":"B1gF56VYPH","keywords":["generation","unsupervised","visualization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep 3D Pan via Local adaptive \"t-shaped\" convolutions with global and local adaptive dilations"},"forum":"B1gF56VYPH","id":"B1gF56VYPH"},{"content":{"TLDR":"Incremental-RNNs resolves exploding/vanishing gradient problem by updating state vectors based on difference between previous state and that predicted by an ODE.","abstract":" ","authors":["Anil Kag","Ziming Zhang","Venkatesh Saligrama"],"iclr_id":"HylpqA4FwS","keywords":["rnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"RNNs Incrementally Evolving on an Equilibrium Manifold: A Panacea for Vanishing and Exploding Gradients?"},"forum":"HylpqA4FwS","id":"HylpqA4FwS"},{"content":{"TLDR":"Generative Adversarial Networks (GANs) are known to be difficult to train, despite considerable research effort. Several regularization techniques for stabilizing training have been proposed, but they introduce non-trivial computational overheads and...","abstract":" ","authors":["Han Zhang","Zizhao Zhang","Augustus Odena","Honglak Lee"],"iclr_id":"S1lxKlSKPH","keywords":["adversarial","gan","generation","image generation","imagenet","regularization","semi supervised learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Consistency Regularization for Generative Adversarial Networks"},"forum":"S1lxKlSKPH","id":"S1lxKlSKPH"},{"content":{"TLDR":"Adversarial examples raise questions about whether neural network models are sensitive to the same visual features as humans. In this paper, we first detect adversarial examples or otherwise corrupted images based on a class-conditional reconstructio...","abstract":" ","authors":["Yao Qin","Nicholas Frosst","Sara Sabour","Colin Raffel","Garrison Cottrell","Geoffrey Hinton"],"iclr_id":"Skgy464Kvr","keywords":["adversarial","adversarial attacks","cnn","perturbation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Detecting and Diagnosing Adversarial Images with Class-Conditional Capsule Reconstructions"},"forum":"Skgy464Kvr","id":"Skgy464Kvr"},{"content":{"TLDR":"A novel 4D CNN structure for video-level representation learning, surpassing  recent 3D CNNs.","abstract":" ","authors":["Shiwen Zhang","Sheng Guo","Weilin Huang","Matthew R. Scott","Limin Wang"],"iclr_id":"SJeLopEYDH","keywords":["cnn","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"V4D: 4D Convolutional Neural Networks for Video-level Representation Learning"},"forum":"SJeLopEYDH","id":"SJeLopEYDH"},{"content":{"TLDR":"A deep hierarchical state-space model in which the state transitions of correlated objects are coordinated by graph neural networks.","abstract":" ","authors":["Fan Yang","Ling Chen","Fan Zhou","Yusong Gao","Wei Cao"],"iclr_id":"B1lGU64tDr","keywords":["graph networks","time series"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Relational State-Space Model for Stochastic Multi-Object Systems"},"forum":"B1lGU64tDr","id":"B1lGU64tDr"},{"content":{"TLDR":"We train a neural network to compute function integrals, and to solve complex differential equations.","abstract":" ","authors":["Guillaume Lample","Fran\u00e7ois Charton"],"iclr_id":"S1eZYeHFDS","keywords":["transformer"],"recs":[],"session":["Wednesday: Symbols and Discovery"],"session_links":[null],"session_times":[null],"title":"Deep Learning For Symbolic Mathematics"},"forum":"S1eZYeHFDS","id":"S1eZYeHFDS"},{"content":{"TLDR":"If warmup is the answer, what is the question?","abstract":" ","authors":["Liyuan Liu","Haoming Jiang","Pengcheng He","Weizhu Chen","Xiaodong Liu","Jianfeng Gao","Jiawei Han"],"iclr_id":"rkgz2aEKDr","keywords":["adam","generalization","language modeling","learning rate","machine translation","nlp","optimization","robustness","variance reduction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"On the Variance of the Adaptive Learning Rate and Beyond"},"forum":"rkgz2aEKDr","id":"rkgz2aEKDr"},{"content":{"TLDR":"We propose a new framework for reasoning about information in complex systems. Our foundation is based on a variational extension of Shannon\u2019s information theory that takes into account the modeling power and computational constraints of the observer...","abstract":" ","authors":["Yilun Xu","Shengjia Zhao","Jiaming Song","Russell Stewart","Stefano Ermon"],"iclr_id":"r1eBeyHFDH","keywords":["information theory","mutual information","reasoning","representation learning","structure learning"],"recs":[],"session":["Wednesday: Theory"],"session_links":[null],"session_times":[null],"title":"A Theory of Usable Information under Computational Constraints"},"forum":"r1eBeyHFDH","id":"r1eBeyHFDH"},{"content":{"TLDR":"We study the phenomenon that some modules of DNNs are more critical than others. Our analysis leads us to propose a complexity measure, that is able to explain the superior generalization performance of some architectures over others.","abstract":" ","authors":["Niladri Chatterji","Behnam Neyshabur","Hanie Sedghi"],"iclr_id":"S1e4jkSKvB","keywords":["generalization","loss landscape"],"recs":[],"session":["Wednesday: Theory"],"session_links":[null],"session_times":[null],"title":"The intriguing role of module criticality in the generalization of deep networks"},"forum":"S1e4jkSKvB","id":"S1e4jkSKvB"},{"content":{"TLDR":"We generate critical states of a trained RL algorithms to visualize potential weaknesses. ","abstract":" ","authors":["Christian Rupprecht","Cyril Ibrahim","Christopher J. Pal"],"iclr_id":"rylvYaNYDH","keywords":["autonomous driving","generative models","reinforcement learning","visualization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Finding and Visualizing Weaknesses of Deep Reinforcement Learning Agents"},"forum":"rylvYaNYDH","id":"rylvYaNYDH"},{"content":{"TLDR":"Generative Models + Federated Learning + Differential Privacy gives data scientists a way to analyze private, decentralized data (e.g., on mobile devices) where direct inspection is prohibited.","abstract":" ","authors":["Sean Augenstein","H. Brendan McMahan","Daniel Ramage","Swaroop Ramaswamy","Peter Kairouz","Mingqing Chen","Rajiv Mathews","Blaise Aguera y Arcas"],"iclr_id":"SJgaRA4FPH","keywords":["federated learning","gan","generative models","privacy","security"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Generative Models for Effective ML on Private, Decentralized Datasets"},"forum":"SJgaRA4FPH","id":"SJgaRA4FPH"},{"content":{"TLDR":"Recurrent neural networks (RNNs) allow an agent to construct a state-representation from a stream of experience, which is essential in partially observable problems. However, there are two primary issues one must overcome when training an RNN: the se...","abstract":" ","authors":["Somjit Nath","Vincent Liu","Alan Chan","Xin Li","Adam White","Martha White"],"iclr_id":"SJgmR0NKPr","keywords":["incremental learning","partial observability","rnn"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Training Recurrent Neural Networks Online by Learning Explicit State Variables"},"forum":"SJgmR0NKPr","id":"SJgmR0NKPr"},{"content":{"TLDR":"We propose procedures for evaluating and strengthening contextual embedding alignment and show that they both improve multilingual BERT's zero-shot XNLI transfer and provide useful insights into the model.","abstract":" ","authors":["Steven Cao","Nikita Kitaev","Dan Klein"],"iclr_id":"r1xCMyBtPS","keywords":["nlp","transfer learning","transformer","word embedding","word embeddings"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Multilingual Alignment of Contextual Word Representations"},"forum":"r1xCMyBtPS","id":"r1xCMyBtPS"},{"content":{"TLDR":"A general framework of value-based reinforcement learning for continuous control","abstract":" ","authors":["Moonkyung Ryu","Yinlam Chow","Ross Anderson","Christian Tjandraatmadja","Craig Boutilier"],"iclr_id":"BkxXe0Etwr","keywords":["clustering","continuous control","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"CAQL: Continuous Action Q-Learning"},"forum":"BkxXe0Etwr","id":"BkxXe0Etwr"},{"content":{"TLDR":"Training agents with adaptive computation based on information bottleneck can promote generalization. ","abstract":" ","authors":["Anirudh Goyal","Yoshua Bengio","Matthew Botvinick","Sergey Levine"],"iclr_id":"Hye1kTVFDS","keywords":["compression","generalization","information bottleneck","optimization","planning","reinforcement learning","variational inference","variational information bottleneck"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"The Variational Bandwidth Bottleneck: Stochastic Evaluation on an Information Budget"},"forum":"Hye1kTVFDS","id":"Hye1kTVFDS"},{"content":{"TLDR":"Autoencoder-based learning has emerged as a staple for disciplining representations in unsupervised and semi-supervised settings. This paper analyzes a framework for improving generalization in a purely supervised setting, where the target space is h...","abstract":" ","authors":["Daniel Jarrett","Mihaela van der Schaar"],"iclr_id":"BygXFkSYDH","keywords":["autoencoder","generalization","regularization","representation learning","stability","unsupervised"],"recs":[],"session":["Tuesday: Representation Learning"],"session_links":[null],"session_times":[null],"title":"Target-Embedding Autoencoders for Supervised Representation Learning"},"forum":"BygXFkSYDH","id":"BygXFkSYDH"},{"content":{"TLDR":"We develop a method for stable offline reinforcement learning from logged data. The key is to regularize the RL policy towards a learned \"advantage weighted\" model of the data.","abstract":" ","authors":["Noah Siegel","Jost Tobias Springenberg","Felix Berkenkamp","Abbas Abdolmaleki","Michael Neunert","Thomas Lampe","Roland Hafner","Nicolas Heess","Martin Riedmiller"],"iclr_id":"rke7geHtwH","keywords":["continuous control","off policy","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Keep Doing What Worked: Behavior Modelling Priors for Offline Reinforcement Learning"},"forum":"rke7geHtwH","id":"rke7geHtwH"},{"content":{"TLDR":"Vector quantization layers incorporated into a self-supervised neural model of speech audio learn hierarchical and discrete linguistic units (phone-like, word-like) when trained with a visual-grounding objective. ","abstract":" ","authors":["David Harwath","Wei-Ning Hsu","James Glass"],"iclr_id":"B1elCp4KwH","keywords":["nlp","quantization","representation learning","robustness","self supervised learning"],"recs":[],"session":["Monday: Signals and Systems"],"session_links":[null],"session_times":[null],"title":"Learning Hierarchical Discrete Linguistic Units from Visually-Grounded Speech"},"forum":"B1elCp4KwH","id":"B1elCp4KwH"},{"content":{"TLDR":"The notion of order learning is proposed and it is applied to regression problems in computer vision","abstract":" ","authors":["Kyungsun Lim","Nyeong-Ho Shin","Young-Yoon Lee","Chang-Su Kim"],"iclr_id":"HygsuaNFwr","keywords":["unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Order Learning and Its Application to Age Estimation"},"forum":"HygsuaNFwr","id":"HygsuaNFwr"},{"content":{"TLDR":"We study the failure modes of DARTS (Differentiable Architecture Search) by looking at the eigenvalues of the Hessian of validation loss w.r.t. the architecture and propose robustifications based on our analysis.","abstract":" ","authors":["Arber Zela","Thomas Elsken","Tonmoy Saikia","Yassine Marrakchi","Thomas Brox","Frank Hutter"],"iclr_id":"H1gDNyrKDS","keywords":["attention","automl","computer vision","generalization","neural architecture search","nlp","optimization","regression","regularization"],"recs":[],"session":["Thursday: Network Architectures"],"session_links":[null],"session_times":[null],"title":"Understanding and Robustifying Differentiable Architecture Search"},"forum":"H1gDNyrKDS","id":"H1gDNyrKDS"},{"content":{"TLDR":"We propose an expansion-based approach for task-free continual learning for the first time. Our model consists of a set of neural network experts and expands the number of experts under the Bayesian nonparametric principle.","abstract":" ","authors":["Soochan Lee","Junsoo Ha","Dongsu Zhang","Gunhee Kim"],"iclr_id":"SJxSOJStPr","keywords":["catastrophic forgetting","continual learning","generation","image generation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Neural Dirichlet Process Mixture Model for Task-Free Continual Learning"},"forum":"SJxSOJStPr","id":"SJxSOJStPr"},{"content":{"TLDR":"Deep neural networks (DNNs) are poorly calibrated when trained in conventional ways. To improve confidence calibration of DNNs, we propose a novel training method, distance-based learning from errors (DBLE). DBLE bases its confidence estimation on di...","abstract":" ","authors":["Chen Xing","Sercan Arik","Zizhao Zhang","Tomas Pfister"],"iclr_id":"BJeB5hVtvB","keywords":["calibration","uncertainty","uncertainty estimation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Distance-Based Learning from Errors for Confidence Calibration"},"forum":"BJeB5hVtvB","id":"BJeB5hVtvB"},{"content":{"TLDR":"Mutual Information (MI) plays an important role in representation learning. However, MI is unfortunately intractable in continuous and high-dimensional settings. Recent advances establish tractable and scalable MI estimators to discover useful repres...","abstract":" ","authors":["Liangjian Wen","Yiji Zhou","Lirong He","Mingyuan Zhou","Zenglin Xu"],"iclr_id":"ByxaUgrFvH","keywords":["information bottleneck","mutual information","representation learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Mutual Information Gradient Estimation for  Representation Learning"},"forum":"ByxaUgrFvH","id":"ByxaUgrFvH"},{"content":{"TLDR":"This paper proposes a meta-learning objective based on speed of adaptation to transfer distributions to discover a modular decomposition and causal variables.","abstract":" ","authors":["Yoshua Bengio","Tristan Deleu","Nasim Rahaman","Nan Rosemary Ke","Sebastien Lachapelle","Olexa Bilaniuk","Anirudh Goyal","Christopher Pal"],"iclr_id":"ryxWIgBFPS","keywords":["causality","meta learning","structure learning","transfer learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"A Meta-Transfer Objective for Learning to Disentangle Causal Mechanisms"},"forum":"ryxWIgBFPS","id":"ryxWIgBFPS"},{"content":{"TLDR":"We present Decentralized Distributed Proximal Policy Optimization (DD-PPO), a method for distributed reinforcement learning in resource-intensive simulated environments. DD-PPO is distributed (uses multiple machines), decentralized (lacks a centraliz...","abstract":" ","authors":["Erik Wijmans","Abhishek Kadian","Ari Morcos","Stefan Lee","Irfan Essa","Devi Parikh","Manolis Savva","Dhruv Batra"],"iclr_id":"H1gX8C4YPr","keywords":["distributed","fine tuning","imagenet","navigation","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"DD-PPO: Learning Near-Perfect PointGoal Navigators from 2.5 Billion Frames"},"forum":"H1gX8C4YPr","id":"H1gX8C4YPr"},{"content":{"TLDR":"We develop a framework to find modular internal representations in generative models and manipulate then to generate counterfactual examples.","abstract":" ","authors":["Michel Besserve","Arash Mehrjou","R\u00e9my Sun","Bernhard Sch\u00f6lkopf"],"iclr_id":"SJxDDpEKvH","keywords":["causality","disentanglement","generalization","generative models","representation learning","robustness","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Counterfactuals uncover the modular structure of deep generative models"},"forum":"SJxDDpEKvH","id":"SJxDDpEKvH"},{"content":{"TLDR":"The space of approximate solutions of CycleGAN admits a lot of symmetry, and an identity loss does not fix this.","abstract":" ","authors":["Nikita Moriakov","Jonas Adler","Jonas Teuwen"],"iclr_id":"B1eWOJHKvB","keywords":["adversarial","generative models","perturbation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Kernel of CycleGAN as a principal homogeneous space"},"forum":"B1eWOJHKvB","id":"B1eWOJHKvB"},{"content":{"TLDR":"Inductive representation learning on temporal graphs is an important step toward salable machine learning on real-world dynamic networks. The evolving nature of temporal dynamic graphs requires handling new nodes as well as capturing temporal pattern...","abstract":" ","authors":["da Xu","chuanwei ruan","evren korpeoglu","sushant kumar","kannan achan"],"iclr_id":"rJeW1yHYwH","keywords":["attention","graph embedding","representation learning","self attention"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Inductive representation learning on temporal graphs"},"forum":"rJeW1yHYwH","id":"rJeW1yHYwH"},{"content":{"TLDR":"Introduce a formal setting for budgeted training and propose a budget-aware linear learning rate schedule","abstract":" ","authors":["Mengtian Li","Ersin Yumer","Deva Ramanan"],"iclr_id":"HyxLRTVKPH","keywords":["imagenet","learning rate","neural architecture search","optimization","semantic segmentation"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Budgeted Training: Rethinking Deep Neural Network Training Under Resource Constraints"},"forum":"HyxLRTVKPH","id":"HyxLRTVKPH"},{"content":{"TLDR":"Differentiable multi-hop access to a textual knowledge base of indexed contextual representations","abstract":" ","authors":["Bhuwan Dhingra","Manzil Zaheer","Vidhisha Balachandran","Graham Neubig","Ruslan Salakhutdinov","William W. Cohen"],"iclr_id":"SJxstlHFPH","keywords":["multi hop qa","nlp","question answering","reasoning"],"recs":[],"session":["Thursday: Optimisation II"],"session_links":[null],"session_times":[null],"title":"Differentiable Reasoning over a Virtual Knowledge Base"},"forum":"SJxstlHFPH","id":"SJxstlHFPH"},{"content":{"TLDR":"While recent continual learning methods largely alleviate the catastrophic problem on toy-sized datasets, there are issues that remain to be tackled in order to apply them to real-world problem domains. First, a continual learning model should effect...","abstract":" ","authors":["Jaehong Yoon","Saehoon Kim","Eunho Yang","Sung Ju Hwang"],"iclr_id":"r1gdj2EKPB","keywords":["catastrophic forgetting","continual learning","fairness","lifelong learning","memory","robustness","scalability"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Scalable and Order-robust Continual Learning with Additive Parameter Decomposition"},"forum":"r1gdj2EKPB","id":"r1gdj2EKPB"},{"content":{"TLDR":"Generative neural models have improved dramatically recently. With this progress comes the risk that such models will be used to attack systems that rely on sensor data for authentication and anomaly detection. Many such learning systems are installe...","abstract":" ","authors":["Roy Mor","Erez Peterfreund","Matan Gavish","Amir Globerson"],"iclr_id":"BkgzMCVtPB","keywords":["anomaly detection","information theory"],"recs":[],"session":["Monday: Generative models"],"session_links":[null],"session_times":[null],"title":"Optimal Strategies Against Generative Attacks"},"forum":"BkgzMCVtPB","id":"BkgzMCVtPB"},{"content":{"TLDR":"This paper extends neural module networks to answer compositional questions against text by introducing differentiable modules that perform reasoning over text and symbols in a probabilistic manner.","abstract":" ","authors":["Nitish Gupta","Kevin Lin","Dan Roth","Sameer Singh","Matt Gardner"],"iclr_id":"SygWvAVFPr","keywords":["compositionality","inductive bias","nlp","question answering","reading comprehension","reasoning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Neural Module Networks for Reasoning over Text"},"forum":"SygWvAVFPr","id":"SygWvAVFPr"},{"content":{"TLDR":"We propose an algorithm combining calibrated prediction and generalization bounds from learning theory to construct confidence sets for deep neural networks with PAC guarantees---i.e., the confidence set for a given input contains the true label with...","abstract":" ","authors":["Sangdon Park","Osbert Bastani","Nikolai Matni","Insup Lee"],"iclr_id":"BJxVI04YvB","keywords":["deep learning theory","generalization","imagenet","regression","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"PAC Confidence Sets for Deep Neural Networks via Calibrated Prediction"},"forum":"BJxVI04YvB","id":"BJxVI04YvB"},{"content":{"TLDR":"We study the most practical problem setup for evaluating adversarial robustness of a machine learning system with limited access:  the hard-label black-box attack setting for generating adversarial examples, where limited model queries are allowed an...","abstract":" ","authors":["Minhao Cheng","Simranjit Singh","Patrick H. Chen","Pin-Yu Chen","Sijia Liu","Cho-Jui Hsieh"],"iclr_id":"SklTQCNtvS","keywords":["adversarial","imagenet","optimization","perturbation","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Sign-OPT: A Query-Efficient Hard-label Adversarial Attack"},"forum":"SklTQCNtvS","id":"SklTQCNtvS"},{"content":{"TLDR":"We propose ensembles of mixed-precision DNNs as a new form of defense against adversarial attacks","abstract":" ","authors":["Sanchari Sen","Balaraman Ravindran","Anand Raghunathan"],"iclr_id":"HJem3yHKwH","keywords":["adversarial","adversarial attacks","ensembles","imagenet","memory","perturbation","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"EMPIR: Ensembles of Mixed Precision Deep Networks for Increased Robustness Against Adversarial Attacks"},"forum":"HJem3yHKwH","id":"HJem3yHKwH"},{"content":{"TLDR":"Better audio synthesis by combining interpretable DSP with end-to-end learning.","abstract":" ","authors":["Jesse Engel","Lamtharn (Hanoi) Hantrakul","Chenjie Gu","Adam Roberts"],"iclr_id":"B1x1ma4tDr","keywords":["adversarial","audio","autoencoder","autoregressive models","disentanglement","expressive power","generation","generative models","inductive bias"],"recs":[],"session":["Monday: Sequence Representations"],"session_links":[null],"session_times":[null],"title":"DDSP: Differentiable Digital Signal Processing"},"forum":"B1x1ma4tDr","id":"B1x1ma4tDr"},{"content":{"TLDR":"This simple problem's optimal value function is fractal and is like a Cantor function.","abstract":" ","authors":["Baoxiang Wang","Shuai Li","Jiajin Li","Siu On Chan"],"iclr_id":"HyxnMyBKwB","keywords":["reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"The Gambler's Problem and Beyond"},"forum":"HyxnMyBKwB","id":"HyxnMyBKwB"},{"content":{"TLDR":"We characterize the expressive power of GNNs in terms of classical logical languages, separating different GNNs and showing connections with standard notions in Knowledge Representation.","abstract":" ","authors":["Pablo Barcel\u00f3","Egor V. Kostylev","Mikael Monet","Jorge P\u00e9rez","Juan Reutter","Juan Pablo Silva"],"iclr_id":"r1lZ7AEKvB","keywords":["graph networks"],"recs":[],"session":["Thursday: Graphs and Representations"],"session_links":[null],"session_times":[null],"title":"The Logical Expressiveness of Graph Neural Networks"},"forum":"r1lZ7AEKvB","id":"r1lZ7AEKvB"},{"content":{"TLDR":"We extend the information bottleneck method to the unsupervised multiview setting and show state of the art results on standard datasets","abstract":" ","authors":["Marco Federici","Anjan Dutta","Patrick Forr\u00e9","Nate Kushman","Zeynep Akata"],"iclr_id":"B1xwcyHFDr","keywords":["generalization","information bottleneck","information theory","representation learning","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning Robust Representations via Multi-View Information Bottleneck"},"forum":"B1xwcyHFDr","id":"B1xwcyHFDr"},{"content":{"TLDR":"A deep neural networks (DNN) with piecewise linear activations can partition the input space into numerous small linear regions, where different linear functions are fitted. It is believed that the number of these regions represents the expressivity ...","abstract":" ","authors":["Xiao Zhang","Dongrui Wu"],"iclr_id":"SkeFl1HKwr","keywords":["optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Empirical Studies on the Properties of Linear Regions in Deep Neural Networks"},"forum":"SkeFl1HKwr","id":"SkeFl1HKwr"},{"content":{"TLDR":"Motivated by the human's ability to continually learn and gain knowledge over time, several research efforts have been pushing the limits of machines to constantly learn while alleviating catastrophic forgetting. Most of the existing methods have bee...","abstract":" ","authors":["Yuanpeng Li","Liang Zhao","Kenneth Church","Mohamed Elhoseiny"],"iclr_id":"rklnDgHtDS","keywords":["catastrophic forgetting","compositionality","continual learning","lifelong learning","machine translation","nlp","sequence modeling"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Compositional Language Continual Learning"},"forum":"rklnDgHtDS","id":"rklnDgHtDS"},{"content":{"TLDR":"We propose a self-supervised learning formulation that simultaneously learns feature representations and useful dataset labels by optimizing the common cross-entropy loss for features _and_ labels, while maximizing information.","abstract":" ","authors":["Asano YM.","Rupprecht C.","Vedaldi A."],"iclr_id":"Hyx-jyBFPr","keywords":["clustering","imagenet","representation learning","self supervision","unsupervised"],"recs":[],"session":["Tuesday: Representation Learning"],"session_links":[null],"session_times":[null],"title":"Self-labelling via simultaneous clustering and representation learning"},"forum":"Hyx-jyBFPr","id":"Hyx-jyBFPr"},{"content":{"TLDR":"We propose a graph sampling based minibatch construction method for training deep Graph Convolutional Networks on large graphs. ","abstract":" ","authors":["Hanqing Zeng","Hongkuan Zhou","Ajitesh Srivastava","Rajgopal Kannan","Viktor Prasanna"],"iclr_id":"BJe8pkHFwS","keywords":["attention","cnn","graph networks","variance reduction"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"GraphSAINT: Graph Sampling Based Inductive Learning Method"},"forum":"BJe8pkHFwS","id":"BJe8pkHFwS"},{"content":{"TLDR":"We propose a model-agnostic approach to explain the behaviour of black-box deep RL agents, trained to play Atari and board games, by highlighting relevant features of an input state.","abstract":" ","authors":["Piyush Gupta","Nikaash Puri","Sukriti Verma","Dhruv Kayastha","Shripad Deshmukh","Balaji Krishnamurthy","Sameer Singh"],"iclr_id":"SJgzLkBKPB","keywords":["interpretability","perturbation","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Explain Your Move: Understanding Agent Actions Using Focused Feature Saliency"},"forum":"SJgzLkBKPB","id":"SJgzLkBKPB"},{"content":{"TLDR":"LP-based upper bounds on the Lipschitz constant of Neural Networks","abstract":" ","authors":["Fabian Latorre","Paul Rolland","Volkan Cevher"],"iclr_id":"rJe4_xSFDB","keywords":["optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Lipschitz constant estimation of Neural Networks via sparse polynomial optimization"},"forum":"rJe4_xSFDB","id":"rJe4_xSFDB"},{"content":{"TLDR":"Our proposed ASN characterizes different actions' influence on other agents using neural networks based on the action semantics between them.","abstract":" ","authors":["Weixun Wang","Tianpei Yang","Yong Liu","Jianye Hao","Xiaotian Hao","Yujing Hu","Yingfeng Chen","Changjie Fan","Yang Gao"],"iclr_id":"ryg48p4tPH","keywords":["multiagent","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Action Semantics Network: Considering the Effects of Actions in Multiagent Systems"},"forum":"ryg48p4tPH","id":"ryg48p4tPH"},{"content":{"TLDR":"The neural tangent kernel in a randomly initialized ReLU net is non-trivial fluctuations as long as the depth and width are comparable. ","abstract":" ","authors":["Boris Hanin","Mihai Nica"],"iclr_id":"SJgndT4KwB","keywords":["neural tangent kernel","relu networks"],"recs":[],"session":["Wednesday: Optimisation I"],"session_links":[null],"session_times":[null],"title":"Finite Depth and Width Corrections to the Neural Tangent Kernel"},"forum":"SJgndT4KwB","id":"SJgndT4KwB"},{"content":{"TLDR":"A novel set of metrics for measuring reliability of reinforcement learning algorithms (+ accompanying statistical tests)","abstract":" ","authors":["Stephanie C.Y. Chan","Samuel Fishman","Anoop Korattikara","John Canny","Sergio Guadarrama"],"iclr_id":"SJlpYJBKvH","keywords":["attention","reinforcement learning","reliability"],"recs":[],"session":["Monday: Reliable RL"],"session_links":[null],"session_times":[null],"title":"Measuring the Reliability of Reinforcement Learning Algorithms"},"forum":"SJlpYJBKvH","id":"SJlpYJBKvH"},{"content":{"TLDR":"We propose a novel method to calibrate knowledge graph embedding models without the need of negative examples.","abstract":" ","authors":["Pedro Tabacof","Luca Costabello"],"iclr_id":"S1g8K1BFwS","keywords":["calibration","graph embedding","graph networks","knowledge graph embeddings","knowledge graphs","regression","representation learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Probability Calibration for Knowledge Graph Embedding Models"},"forum":"S1g8K1BFwS","id":"S1g8K1BFwS"},{"content":{"TLDR":"Using a structured quantization technique aiming at better in-domain reconstruction to compress convolutional neural networks","abstract":" ","authors":["Pierre Stock","Armand Joulin","R\u00e9mi Gribonval","Benjamin Graham","Herv\u00e9 J\u00e9gou"],"iclr_id":"rJehVyrKwH","keywords":["compression","imagenet","memory","quantization"],"recs":[],"session":["Thursday: Network Architectures"],"session_links":[null],"session_times":[null],"title":"And the Bit Goes Down: Revisiting the Quantization of Neural Networks"},"forum":"rJehVyrKwH","id":"rJehVyrKwH"},{"content":{"TLDR":"We describe a biologically inspired sleep algorithm for increasing an artificial neural network's ability to extract the gist of a training set and exhibit increased robustness to adversarial attacks and general distortions.","abstract":" ","authors":["Timothy Tadros","Giri Krishnan","Ramyaa Ramyaa","Maxim Bazhenov"],"iclr_id":"r1xGnA4Kvr","keywords":["adversarial","adversarial attacks","adversarial defense","fine tuning","generalization","perturbation","robustness","security"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Biologically inspired sleep algorithm for increased generalization and adversarial robustness in deep neural networks"},"forum":"r1xGnA4Kvr","id":"r1xGnA4Kvr"},{"content":{"TLDR":"Our approach is the first attempt to leverage a sequential latent variable model for knowledge selection in the multi-turn knowledge-grounded dialogue. It achieves the new state-of-the-art performance on Wizard of Wikipedia benchmark.","abstract":" ","authors":["Byeongchang Kim","Jaewoo Ahn","Gunhee Kim"],"iclr_id":"Hke0K1HKwr","keywords":["generation","nlp","transformer"],"recs":[],"session":["Thursday: Natural Language"],"session_links":[null],"session_times":[null],"title":"Sequential Latent Knowledge Selection for Knowledge-Grounded Dialogue"},"forum":"Hke0K1HKwr","id":"Hke0K1HKwr"},{"content":{"TLDR":"Stochastic style transfer with adjustable features. ","abstract":" ","authors":["Mohammad Babaeizadeh","Golnaz Ghiasi"],"iclr_id":"HJe_Z04Yvr","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Adjustable Real-time Style Transfer"},"forum":"HJe_Z04Yvr","id":"HJe_Z04Yvr"},{"content":{"TLDR":"Deterministic regularized autoencoders can learn a smooth, meaningful latent space as VAEs without having to force some arbitrarily chosen prior (i.e., Gaussian).","abstract":" ","authors":["Partha Ghosh","Mehdi S. M. Sajjadi","Antonio Vergari","Michael Black","Bernhard Scholkopf"],"iclr_id":"S1g7tpEYDS","keywords":["autoencoder","generative models","regularization","unsupervised","variational autoencoders"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"From Variational to Deterministic Autoencoders"},"forum":"S1g7tpEYDS","id":"S1g7tpEYDS"},{"content":{"TLDR":"Estimator for the Wasserstein natural gradient","abstract":" ","authors":["M Arbel","A Gretton","W Li","G Montufar"],"iclr_id":"Hklz71rYvS","keywords":["information geometry","optimization","wasserstein metric"],"recs":[],"session":["Thursday: Optimisation II"],"session_links":[null],"session_times":[null],"title":"Kernelized Wasserstein Natural Gradient"},"forum":"Hklz71rYvS","id":"Hklz71rYvS"},{"content":{"TLDR":"Deep neural networks (DNNs) can be huge in size, requiring a considerable a mount of energy and computational resources to operate, which limits their applications in numerous scenarios. It is thus of interest to compress DNNs while maintaining their...","abstract":" ","authors":["Xin Xing","Long Sha","Pengyu Hong","Zuofeng Shang","Jun S. Liu"],"iclr_id":"HJgCF0VFwr","keywords":["compression","pruning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Probabilistic Connection Importance Inference and Lossless Compression of Deep Neural Networks"},"forum":"HJgCF0VFwr","id":"HJgCF0VFwr"},{"content":{"TLDR":"A zero-shot segmentation framework for 3D object part segmentation. Model the segmentation as a decision-making process and solve as a contextual bandit problem.","abstract":" ","authors":["Tiange Luo","Kaichun Mo","Zhiao Huang","Jiarui Xu","Siyu Hu","Liwei Wang","Hao Su"],"iclr_id":"rkl8dlHYvB","keywords":["zero shot learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning to Group: A Bottom-Up Framework for 3D Part Discovery in Unseen Categories"},"forum":"rkl8dlHYvB","id":"rkl8dlHYvB"},{"content":{"TLDR":"We propose Symplectic Recurrent Neural Networks (SRNNs) as learning algorithms that capture the dynamics of physical systems from observed trajectories. SRNNs model the Hamiltonian function of the system by a neural networks, and leverage symplectic ...","abstract":" ","authors":["Zhengdao Chen","Jianyu Zhang","Martin Arjovsky","L\u00e9on Bottou"],"iclr_id":"BkgYPREtPr","keywords":["optimization","rnn"],"recs":[],"session":["Wednesday: Sequence Representations"],"session_links":[null],"session_times":[null],"title":"Symplectic Recurrent Neural Networks"},"forum":"BkgYPREtPr","id":"BkgYPREtPr"},{"content":{"TLDR":"We propose the first algorithm for verifying the robustness of Transformers.","abstract":" ","authors":["Zhouxing Shi","Huan Zhang","Kai-Wei Chang","Minlie Huang","Cho-Jui Hsieh"],"iclr_id":"BJxwPJHFwS","keywords":["attention","interval bound propagation","robustness","self attention","transformer","verification"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Robustness Verification for Transformers"},"forum":"BJxwPJHFwS","id":"BJxwPJHFwS"},{"content":{"TLDR":"State and action embeddings which incorporate the dynamics improve exploration and RL from pixels.","abstract":" ","authors":["William Whitney","Rajat Agarwal","Kyunghyun Cho","Abhinav Gupta"],"iclr_id":"BJgZGeHFPH","keywords":["continuous control","reinforcement learning","representation learning","sample efficiency"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Dynamics-Aware Embeddings"},"forum":"BJgZGeHFPH","id":"BJgZGeHFPH"},{"content":{"TLDR":"We prove that there exist ReLU networks whose parameters are almost uniquely determined by the function they implement.","abstract":" ","authors":["Mary Phuong","Christoph H. Lampert"],"iclr_id":"Bylx-TNKvH","keywords":["over parameterization","relu networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Functional vs. parametric equivalence of ReLU networks"},"forum":"Bylx-TNKvH","id":"Bylx-TNKvH"},{"content":{"TLDR":"A novel, non-adversarial, approach to learn latent variable models in general and mixture models in particular by computing the I-Projection solely based on samples.","abstract":" ","authors":["Philipp Becker","Oleg Arenz","Gerhard Neumann"],"iclr_id":"ByglLlHFDS","keywords":["density estimation","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Expected Information Maximization: Using the I-Projection for Mixture Density Estimation"},"forum":"ByglLlHFDS","id":"ByglLlHFDS"},{"content":{"TLDR":"A fast optimizer for general applications and large-batch training.","abstract":" ","authors":["Yang You","Jing Li","Sashank Reddi","Jonathan Hseu","Sanjiv Kumar","Srinadh Bhojanapalli","Xiaodan Song","James Demmel","Kurt Keutzer","Cho-Jui Hsieh"],"iclr_id":"Syx4wnEtvH","keywords":["attention","distributed","imagenet","learning rate","memory","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes"},"forum":"Syx4wnEtvH","id":"Syx4wnEtvH"},{"content":{"TLDR":"An end-to-end trainable model compression method optimizing accuracy jointly with the expected model size.","abstract":" ","authors":["Deniz Oktay","Johannes Ball\u00e9","Saurabh Singh","Abhinav Shrivastava"],"iclr_id":"HkgxW0EYDS","keywords":["compression","computer vision","imagenet","information theory","model compression"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Scalable Model Compression by Entropy Penalized Reparameterization"},"forum":"HkgxW0EYDS","id":"HkgxW0EYDS"},{"content":{"TLDR":"We provide theoretical results about the effect of activation function on the training of highly overparametrized 2-layer neural networks","abstract":" ","authors":["Abhishek Panigrahi","Abhishek Shetty","Navin Goyal"],"iclr_id":"rkgfdeBYvH","keywords":["deep learning theory"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Effect of Activation Functions on the Training of Overparametrized Neural Nets"},"forum":"rkgfdeBYvH","id":"rkgfdeBYvH"},{"content":{"TLDR":"New Objective for One-Shot Pruning Recurrent Neural Networks","abstract":" ","authors":["Shunshi Zhang","Bradly C. Stadie"],"iclr_id":"r1e9GCNKvH","keywords":["cnn","pruning","rnn","sparsity"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"One-Shot Pruning of Recurrent Neural Networks by Jacobian Spectrum Evaluation"},"forum":"r1e9GCNKvH","id":"r1e9GCNKvH"},{"content":{"TLDR":"We present a new hierarchical reinforcement learning algorithm which can solve high-dimensional goal-oriented tasks  more reliably than non-hierarchical agents and other state-of-the-art skill discovery techniques.","abstract":" ","authors":["Akhil Bagaria","George Konidaris"],"iclr_id":"B1gqipNYwH","keywords":["continuous control","hierarchical reinforcement learning","reinforcement learning","skill discovery"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Option Discovery using Deep Skill Chaining"},"forum":"B1gqipNYwH","id":"B1gqipNYwH"},{"content":{"TLDR":"When constructing random forests, it is of prime importance to ensure high accuracy and low correlation of individual tree classifiers for good performance. Nevertheless, it is typically difficult for existing random forest methods to strike a good b...","abstract":" ","authors":["Jongbin Ryu","Gitaek Kwon","Ming-Hsuan Yang","Jongwoo Lim"],"iclr_id":"H1lxVyStPH","keywords":["generalization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Generalized Convolutional Forest Networks for Domain Generalization and Visual Recognition"},"forum":"H1lxVyStPH","id":"H1lxVyStPH"},{"content":{"TLDR":"A key challenge of existing program synthesizers is ensuring that the synthesized program generalizes well. This can be difficult to achieve as the specification provided by the end user is often limited, containing as few as one or two input-output ...","abstract":" ","authors":["Larissa Laich","Pavol Bielik","Martin Vechev"],"iclr_id":"BJl07ySKvS","keywords":["program synthesis"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Guiding Program Synthesis by Learning to Generate Examples"},"forum":"BJl07ySKvS","id":"BJl07ySKvS"},{"content":{"TLDR":"Sequential word order is important when processing text. Currently, neural networks (NNs) address this by modeling word position using position embeddings. The problem is that position embeddings capture the position of individual words, but not the ...","abstract":" ","authors":["Benyou Wang","Donghao Zhao","Christina Lioma","Qiuchi Li","Peng Zhang","Jakob Grue Simonsen"],"iclr_id":"Hke-WTVtwr","keywords":["language modeling","machine translation","nlp","text classification","transformer","word embedding","word embeddings"],"recs":[],"session":["Wednesday: Sequence Representations"],"session_links":[null],"session_times":[null],"title":"Encoding word order in complex embeddings"},"forum":"Hke-WTVtwr","id":"Hke-WTVtwr"},{"content":{"TLDR":"Scalability in terms of object density in a scene is a primary challenge in unsupervised sequential object-oriented representation learning. Most of the previous models have been shown to work only on scenes with a few objects. In this paper, we prop...","abstract":" ","authors":["Jindong Jiang","Sepehr Janghorbani","Gerard De Melo","Sungjin Ahn"],"iclr_id":"SJxrKgStDH","keywords":["attention","representation learning","scalability","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"SCALOR: Generative World Models with Scalable Object Representations"},"forum":"SJxrKgStDH","id":"SJxrKgStDH"},{"content":{"TLDR":"We supervise graph neural networks to imitate intermediate and step-wise outputs of classical graph algorithms, recovering highly favourable insights.","abstract":" ","authors":["Petar Veli\u010dkovi\u0107","Rex Ying","Matilde Padovano","Raia Hadsell","Charles Blundell"],"iclr_id":"SkgKO0EtvS","keywords":["graph networks","learning to execute","program synthesis"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Neural Execution of Graph Algorithms"},"forum":"SkgKO0EtvS","id":"SkgKO0EtvS"},{"content":{"TLDR":"Formal verification of a specification on a model's prediction undersensitivity using Interval Bound Propagation","abstract":" ","authors":["Johannes Welbl","Po-Sen Huang","Robert Stanforth","Sven Gowal","Krishnamurthy (Dj) Dvijotham","Martin Szummer","Pushmeet Kohli"],"iclr_id":"SyxhVkrYvr","keywords":["adversarial","attention","interval bound propagation","nlp","robustness","verification"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Towards Verified Robustness under Text Deletion Interventions"},"forum":"SyxhVkrYvr","id":"SyxhVkrYvr"},{"content":{"TLDR":"Graph-structured data is prevalent in many domains. Despite the widely celebrated success of deep neural networks, their power in graph-structured data is yet to be fully explored. We propose a novel network architecture that incorporates advanced gr...","abstract":" ","authors":["Ze Ye","Kin Sum Liu","Tengfei Ma","Jie Gao","Chao Chen"],"iclr_id":"BylEqnVFDB","keywords":["graph networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Curvature Graph Network"},"forum":"BylEqnVFDB","id":"BylEqnVFDB"},{"content":{"TLDR":"We propose physics-aware difference graph networks designed to effectively learn spatial differences to modeling sparsely-observed dynamics.","abstract":" ","authors":["Sungyong Seo","Chuizheng Meng","Yan Liu"],"iclr_id":"r1gelyrtwH","keywords":["distributed","graph networks"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Physics-aware Difference Graph Networks for Sparsely-Observed Dynamics"},"forum":"r1gelyrtwH","id":"r1gelyrtwH"},{"content":{"TLDR":"We verify neural tangent kernel is powerful on small data via experiments on UCI datasets, small CIFAR 10 and low-shot learning on VOC07.","abstract":" ","authors":["Sanjeev Arora","Simon S. Du","Zhiyuan Li","Ruslan Salakhutdinov","Ruosong Wang","Dingli Yu"],"iclr_id":"rkl8sJBYvH","keywords":["deep learning theory","fewshot learning","gradient descent","imagenet","learning rate","neural tangent kernel","regression","transfer learning"],"recs":[],"session":["Tuesday: Representation Learning"],"session_links":[null],"session_times":[null],"title":"Harnessing the Power of Infinitely Wide Deep Nets on Small-data Tasks"},"forum":"rkl8sJBYvH","id":"rkl8sJBYvH"},{"content":{"TLDR":"In this paper, a new population-guided parallel learning scheme is proposed to enhance the performance of off-policy reinforcement learning (RL). In the proposed scheme, multiple identical learners with their own value-functions and policies share a ...","abstract":" ","authors":["Whiyoung Jung","Giseung Park","Youngchul Sung"],"iclr_id":"rJeINp4KwH","keywords":["policy gradient","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Population-Guided Parallel Policy Search for Reinforcement Learning"},"forum":"rJeINp4KwH","id":"rJeINp4KwH"},{"content":{"TLDR":"We demonstrate, and characterize, realistic settings where bigger models are worse, and more data hurts.","abstract":" ","authors":["Preetum Nakkiran","Gal Kaplun","Yamini Bansal","Tristan Yang","Boaz Barak","Ilya Sutskever"],"iclr_id":"B1g5sA4twr","keywords":["gradient descent","optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Deep Double Descent: Where Bigger Models and More Data Hurt"},"forum":"B1g5sA4twr","id":"B1g5sA4twr"},{"content":{"TLDR":"How to prevent stale gradients (in asynchronous SGD) from changing minima stability and degrade steady state generalization?","abstract":" ","authors":["Niv Giladi","Mor Shpigel Nacson","Elad Hoffer","Daniel Soudry"],"iclr_id":"Bkeb7lHtvH","keywords":["generalization","gradient descent","implicit bias","learning rate","momentum","stability"],"recs":[],"session":["Monday: Optimisation Theory"],"session_links":[null],"session_times":[null],"title":"At Stability's Edge: How to Adjust Hyperparameters to Preserve Minima Selection in Asynchronous Training of Neural Networks?"},"forum":"Bkeb7lHtvH","id":"Bkeb7lHtvH"},{"content":{"TLDR":"A framework for evaluating model performance when even experts disagree on what the ground truth is.","abstract":" ","authors":["Igor Lovchinsky","Alon Daks","Israel Malkin","Pouya Samangouei","Ardavan Saeedi","Yang Liu","Swami Sankaranarayanan","Tomer Gafner","Ben Sternlieb","Patrick Maher","Nathan Silberman"],"iclr_id":"Byg-wJSYDS","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Discrepancy Ratio: Evaluating Model Performance When Even Experts Disagree on the Truth"},"forum":"Byg-wJSYDS","id":"Byg-wJSYDS"},{"content":{"TLDR":"We propose a novel 8-bit format that eliminates the need for loss scaling, stochastic rounding, and other low precision techniques","abstract":" ","authors":["Leopold Cambier","Anahita Bhiwandiwalla","Ting Gong","Oguz H. Elibol","Mehran Nekuii","Hanlin Tang"],"iclr_id":"Bkxe2AVtPS","keywords":["fine tuning","memory","quantization","transformer"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Shifted and Squeezed 8-bit Floating Point format for Low-Precision Training of Deep Neural Networks"},"forum":"Bkxe2AVtPS","id":"Bkxe2AVtPS"},{"content":{"TLDR":"We apply a reinforcement learning based approach to learning optimal synchronization policies used for Parameter Server-based distributed training  of SGD.","abstract":" ","authors":["Rong Zhu","Sheng Yang","Andreas Pfadler","Zhengping Qian","Jingren Zhou"],"iclr_id":"rJxX8T4Kvr","keywords":["distributed","gradient descent","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning Efficient Parameter Server Synchronization Policies for Distributed SGD"},"forum":"rJxX8T4Kvr","id":"rJxX8T4Kvr"},{"content":{"TLDR":"Artificial neural networks trained with gradient descent are capable of recapitulating both realistic neural activity and the anatomical organization of a biological circuit.","abstract":" ","authors":["Christopher J. Cueva","Peter Y. Wang","Matthew Chin","Xue-Xin Wei"],"iclr_id":"HklSeREtPB","keywords":["neural circuits","optimization","rnn"],"recs":[],"session":["Tuesday: Biology and ML"],"session_links":[null],"session_times":[null],"title":"Emergence of functional and structural properties of the head direction system by optimization of recurrent neural networks"},"forum":"HklSeREtPB","id":"HklSeREtPB"},{"content":{"TLDR":"Reinforcement Learning and Adaptive Sampling for Optimized Compilation of Deep Neural Networks.","abstract":" ","authors":["Byung Hoon Ahn","Prannoy Pilligundla","Amir Yazdanbakhsh","Hadi Esmaeilzadeh"],"iclr_id":"rygG4AVFvH","keywords":["combinatorial optimization","learning for systems","optimization","reinforcement learning"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Chameleon: Adaptive Code Optimization for Expedited Deep Neural Network Compilation"},"forum":"rygG4AVFvH","id":"rygG4AVFvH"},{"content":{"TLDR":" We propose a representation learning model called Space2vec to encode the absolute positions and spatial relationships of places.","abstract":" ","authors":["Gengchen Mai","Krzysztof Janowicz","Bo Yan","Rui Zhu","Ling Cai","Ni Lao"],"iclr_id":"rJljdh4KDH","keywords":["representation learning","unsupervised"],"recs":[],"session":["Tuesday: Feature Discovery for Structured Data"],"session_links":[null],"session_times":[null],"title":"Multi-Scale Representation Learning  for Spatial Feature Distributions using Grid Cells"},"forum":"rJljdh4KDH","id":"rJljdh4KDH"},{"content":{"TLDR":"We propose a novel method for unsupervised image-to-image translation, which incorporates a new attention module and a new learnable normalization function in an end-to-end manner. The attention module guides our model to focus on more important regi...","abstract":" ","authors":["Junho Kim","Minjae Kim","Hyeonwoo Kang","Kwang Hee Lee"],"iclr_id":"BJlZ5ySKPH","keywords":["attention","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"U-GAT-IT: Unsupervised Generative Attentional Networks with Adaptive Layer-Instance Normalization for Image-to-Image Translation"},"forum":"BJlZ5ySKPH","id":"BJlZ5ySKPH"},{"content":{"TLDR":"We use video prediction models, a model-based reinforcement learning algorithm and 2h of gameplay per game to train agents for 26 Atari games.","abstract":" ","authors":["\u0141ukasz Kaiser","Mohammad Babaeizadeh","Piotr Mi\u0142os","B\u0142a\u017cej Osi\u0144ski","Roy H Campbell","Konrad Czechowski","Dumitru Erhan","Chelsea Finn","Piotr Kozakowski","Sergey Levine","Afroz Mohiuddin","Ryan Sepassi","George Tucker","Henryk Michalewski"],"iclr_id":"S1xCPJHtDB","keywords":["model based reinforcement learning","model based rl","reinforcement learning","video prediction"],"recs":[],"session":["Wednesday: RL and Planning"],"session_links":[null],"session_times":[null],"title":"Model Based Reinforcement Learning for Atari"},"forum":"S1xCPJHtDB","id":"S1xCPJHtDB"},{"content":{"TLDR":"Novel variant of learning to learn framework for zeroth-order optimization that learns both the update rule and the Gaussian sampling rule.","abstract":" ","authors":["Yangjun Ruan","Yuanhao Xiong","Sashank Reddi","Sanjiv Kumar","Cho-Jui Hsieh"],"iclr_id":"ryxz8CVYDH","keywords":["adversarial","black box adversarial attack","meta learning","optimization","zeroth order optimization"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Learning to Learn by Zeroth-Order Oracle"},"forum":"ryxz8CVYDH","id":"ryxz8CVYDH"},{"content":{"TLDR":"Learned joint representations of images and text form the backbone of several important cross-domain tasks such as image captioning. Prior work mostly maps both domains into a common latent representation in a purely supervised fashion. This is rathe...","abstract":" ","authors":["Shweta Mahajan","Iryna Gurevych","Stefan Roth"],"iclr_id":"SJxE8erKDH","keywords":[],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Latent Normalizing Flows for Many-to-Many Cross-Domain Mappings"},"forum":"SJxE8erKDH","id":"SJxE8erKDH"},{"content":{"TLDR":"We propose a novel node embedding of directed graphs to statistical manifolds and analyze connections to divergence, geometry and efficient learning procedure.","abstract":" ","authors":["Thorben Funke","Tian Guo","Alen Lancic","Nino Antulov-Fantulin"],"iclr_id":"SkxQp1StDH","keywords":["graph embedding","information geometry","unsupervised"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Low-dimensional statistical manifold embedding of directed graphs"},"forum":"SkxQp1StDH","id":"SkxQp1StDH"},{"content":{"TLDR":"A framework for incorporating robustness to model misspecification into continuous control Reinforcement Learning algorithms.","abstract":" ","authors":["Daniel J. Mankowitz","Nir Levine","Rae Jeong","Abbas Abdolmaleki","Jost Tobias Springenberg","Yuanyuan Shi","Jackie Kay","Todd Hester","Timothy Mann","Martin Riedmiller"],"iclr_id":"HJgC60EtwB","keywords":["continuous control","optimization","perturbation","reinforcement learning","robustness"],"recs":[],"session":[],"session_links":[],"session_times":[],"title":"Robust Reinforcement Learning for Continuous Control with Model Misspecification"},"forum":"HJgC60EtwB","id":"HJgC60EtwB"}]