People

@article{DBLP:journals/corr/abs-2206-04496,

title = {Mitigating Modality Collapse in Multimodal VAEs via Impartial Optimization},

author = {Adrián Javaloy and Maryam Meghdadi and Isabel Valera},

url = {https://doi.org/10.48550/arXiv.2206.04496},

doi = {10.48550/arXiv.2206.04496},

year  = {2022},

date = {2022-01-01},

urldate = {2022-01-01},

journal = {CoRR},

volume = {abs/2206.04496},

abstract = {A number of variational autoencoders (VAEs) have recently emerged with the aim of modeling multimodal data, e.g., to jointly model images and their corresponding captions. Still, multimodal VAEs tend to focus solely on a subset of the modalities, e.g., by fitting the image while neglecting the caption. We refer to this limitation as modality collapse. In this work, we argue that this effect is a consequence of conflicting gradients during multimodal VAE training. We show how to detect the sub-graphs in the computational graphs where gradients conflict (impartiality blocks), as well as how to leverage existing gradient-conflict solutions from multitask learning to mitigate modality collapse. That is, to ensure impartial optimization across modalities. We apply our training framework to several multimodal VAE models, losses and datasets from the literature, and empirically show that our framework significantly improves the reconstruction performance, conditional generation, and coherence of the latent space across modalities.},

keywords = {adrian, isabel, maryam, project-robustgenerative, spotlight, variational autoencoder},

pubstate = {published},

tppubtype = {article}

}

@article{DBLP:journals/jmlr/ValeraPLG20,

title = {General Latent Feature Models for Heterogeneous Datasets},

author = {Isabel Valera and Melanie F. Pradier and Maria Lomeli and Zoubin Ghahramani},

url = {http://jmlr.org/papers/v21/17-328.html},

year  = {2020},

date = {2020-01-01},

urldate = {2020-01-01},

journal = {J. Mach. Learn. Res.},

volume = {21},

pages = {100:1--100:49},

abstract = {Latent variable models allow capturing the hidden structure underlying the data. In particular, feature allocation models represent each observation by a linear combination of latent variables. These models are often used to make predictions either for new observations or for missing information in the original data, as well as to perform exploratory data analysis. Although there is an extensive literature on latent feature allocation models for homogeneous datasets, where all the attributes that describe each object are of the same (continuous or discrete) type, there is no general framework for practical latent feature modeling for heterogeneous datasets. In this paper, we introduce a general Bayesian nonparametric latent feature allocation model suitable for heterogeneous datasets, where the attributes describing each object can be arbitrary combinations of real-valued, positive real-valued, categorical, ordinal and count variables. The proposed model presents several important properties. First, it is suitable for heterogeneous data while keeping the properties of conjugate models, which enables us to develop an inference algorithm that presents linear complexity with respect to the number of objects and attributes per MCMC iteration. Second, the Bayesian nonparametric component allows us to place a prior distribution on the number of features required to capture the latent structure in the data. Third, the latent features in the model are binary-valued, which facilitates the interpretability of the obtained latent features in exploratory data analysis. Finally, a software package, called GLFM toolbox, is made publicly available for other researchers to use and extend. It is available at https://ivaleram.github.io/GLFM/. We show the flexibility of the proposed model by solving both prediction and data analysis tasks on several real-world datasets.},

keywords = {isabel, project-robustgenerative},

pubstate = {published},

tppubtype = {article}

}