Merge pull request #70 from ptimons44/main

Final Project Submission Patrick Timons
deep-learning-mit · Dec 16, 2023 · 5a7738b · 5a7738b
2 parents 62b1091 + e2f5ab7
commit 5a7738b
Show file tree

Hide file tree

Showing 17 changed files with 254 additions and 2 deletions.
diff --git a/_posts/2023-12-12-Recovering Latent Variables with VAEs despite Training Bias.md b/_posts/2023-12-12-Recovering Latent Variables with VAEs despite Training Bias.md
diff --git a/...s/bibliography/2023-12-12-Recovering Latent Variables with VAEs despite Training Bias.bib b/...s/bibliography/2023-12-12-Recovering Latent Variables with VAEs despite Training Bias.bib
@@ -0,0 +1,86 @@
+@inproceedings{
+higgins2017betavae,
+title={beta-{VAE}: Learning Basic Visual Concepts with a Constrained Variational Framework},
+author={Irina Higgins and Loic Matthey and Arka Pal and Christopher Burgess and Xavier Glorot and Matthew Botvinick and Shakir Mohamed and Alexander Lerchner},
+booktitle={International Conference on Learning Representations},
+year={2017},
+url={https://openreview.net/forum?id=Sy2fzU9gl}
+}
+@inproceedings{
+miao2022on,
+title={On Incorporating Inductive Biases into {VAE}s},
+author={Ning Miao and Emile Mathieu and Siddharth N and Yee Whye Teh and Tom Rainforth},
+booktitle={International Conference on Learning Representations},
+year={2022},
+url={https://openreview.net/forum?id=nzvbBD_3J-g}
+}
+@misc{belghazi2021mine,
+      title={MINE: Mutual Information Neural Estimation}, 
+      author={Mohamed Ishmael Belghazi and Aristide Baratin and Sai Rajeswar and Sherjil Ozair and Yoshua Bengio and Aaron Courville and R Devon Hjelm},
+      year={2021},
+      eprint={1801.04062},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@misc{louizos2017causal,
+      title={Causal Effect Inference with Deep Latent-Variable Models}, 
+      author={Christos Louizos and Uri Shalit and Joris Mooij and David Sontag and Richard Zemel and Max Welling},
+      year={2017},
+      eprint={1705.08821},
+      archivePrefix={arXiv},
+      primaryClass={stat.ML}
+}
+@inproceedings{10.1145/3287560.3287564,
+author = {Madras, David and Creager, Elliot and Pitassi, Toniann and Zemel, Richard},
+title = {Fairness through Causal Awareness: Learning Causal Latent-Variable Models for Biased Data},
+year = {2019},
+isbn = {9781450361255},
+publisher = {Association for Computing Machinery},
+address = {New York, NY, USA},
+url = {https://doi.org/10.1145/3287560.3287564},
+doi = {10.1145/3287560.3287564},
+abstract = {How do we learn from biased data? Historical datasets often reflect historical prejudices; sensitive or protected attributes may affect the observed treatments and outcomes. Classification algorithms tasked with predicting outcomes accurately from these datasets tend to replicate these biases. We advocate a causal modeling approach to learning from biased data, exploring the relationship between fair classification and intervention. We propose a causal model in which the sensitive attribute confounds both the treatment and the outcome. Building on prior work in deep learning and generative modeling, we describe how to learn the parameters of this causal model from observational data alone, even in the presence of unobserved confounders. We show experimentally that fairness-aware causal modeling provides better estimates of the causal effects between the sensitive attribute, the treatment, and the outcome. We further present evidence that estimating these causal effects can help learn policies that are both more accurate and fair, when presented with a historically biased dataset.},
+booktitle = {Proceedings of the Conference on Fairness, Accountability, and Transparency},
+pages = {349–358},
+numpages = {10},
+keywords = {variational inference, causal inference, fairness in machine learning},
+location = {Atlanta, GA, USA},
+series = {FAT* '19}
+}
+@inproceedings{
+floto2023the,
+title={The Tilted Variational Autoencoder: Improving Out-of-Distribution Detection},
+author={Griffin Floto and Stefan Kremer and Mihai Nica},
+booktitle={The Eleventh International Conference on Learning Representations },
+year={2023},
+url={https://openreview.net/forum?id=YlGsTZODyjz}
+}
+@misc{xiao2020likelihood,
+      title={Likelihood Regret: An Out-of-Distribution Detection Score For Variational Auto-encoder}, 
+      author={Zhisheng Xiao and Qing Yan and Yali Amit},
+      year={2020},
+      eprint={2003.02977},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@article{RAN2022199,
+title = {Detecting out-of-distribution samples via variational auto-encoder with reliable uncertainty estimation},
+journal = {Neural Networks},
+volume = {145},
+pages = {199-208},
+year = {2022},
+issn = {0893-6080},
+doi = {https://doi.org/10.1016/j.neunet.2021.10.020},
+url = {https://www.sciencedirect.com/science/article/pii/S0893608021004111},
+author = {Xuming Ran and Mingkun Xu and Lingrui Mei and Qi Xu and Quanying Liu},
+keywords = {Variational auto-encoder, Out-of-distribution detection, Uncertainty estimation, Noise contrastive prior},
+abstract = {Variational autoencoders (VAEs) are influential generative models with rich representation capabilities from the deep neural network architecture and Bayesian method. However, VAE models have a weakness that assign a higher likelihood to out-of-distribution (OOD) inputs than in-distribution (ID) inputs. To address this problem, a reliable uncertainty estimation is considered to be critical for in-depth understanding of OOD inputs. In this study, we propose an improved noise contrastive prior (INCP) to be able to integrate into the encoder of VAEs, called INCPVAE. INCP is scalable, trainable and compatible with VAEs, and it also adopts the merits from the INCP for uncertainty estimation. Experiments on various datasets demonstrate that compared to the standard VAEs, our model is superior in uncertainty estimation for the OOD data and is robust in anomaly detection tasks. The INCPVAE model obtains reliable uncertainty estimation for OOD inputs and solves the OOD problem in VAE models.}
+}
+@misc{kim2019disentangling,
+      title={Disentangling by Factorising}, 
+      author={Hyunjik Kim and Andriy Mnih},
+      year={2019},
+      eprint={1802.05983},
+      archivePrefix={arXiv},
+      primaryClass={stat.ML}
+}
diff --git a/...ring Latent Variables with VAEs despite Training Bias/projection head training curve.html b/...ring Latent Variables with VAEs despite Training Bias/projection head training curve.html
@@ -0,0 +1 @@
+<iframe src="https://wandb.ai/nlp-final-project/Learning%20RV%20to%20Latent%20Linear%20Projection?workspace=user-patricktimons" style="border:none;height:1024px;width:100%"></iframe>
diff --git a/... Variables with VAEs despite Training Bias/Linear Projection Training Curve.png b/... Variables with VAEs despite Training Bias/Linear Projection Training Curve.png
diff --git a/...Latent Variables with VAEs despite Training Bias/MSE generating on test set.png b/...Latent Variables with VAEs despite Training Bias/MSE generating on test set.png
diff --git a/...vering Latent Variables with VAEs despite Training Bias/MSE projection head.png b/...vering Latent Variables with VAEs despite Training Bias/MSE projection head.png
diff --git a/...-Recovering Latent Variables with VAEs despite Training Bias/approximate MI.png b/...-Recovering Latent Variables with VAEs despite Training Bias/approximate MI.png
diff --git a/...-12-12-Recovering Latent Variables with VAEs despite Training Bias/beta = 1.png b/...-12-12-Recovering Latent Variables with VAEs despite Training Bias/beta = 1.png
diff --git a/...12-12-Recovering Latent Variables with VAEs despite Training Bias/beta = 10.png b/...12-12-Recovering Latent Variables with VAEs despite Training Bias/beta = 10.png
diff --git a/...-12-12-Recovering Latent Variables with VAEs despite Training Bias/beta = 5.png b/...-12-12-Recovering Latent Variables with VAEs despite Training Bias/beta = 5.png
diff --git a/...tent Variables with VAEs despite Training Bias/beta=1 mu=[0,2] train curves.png b/...tent Variables with VAEs despite Training Bias/beta=1 mu=[0,2] train curves.png
diff --git a/...nt Variables with VAEs despite Training Bias/beta=1, mu=[2] training curves.png b/...nt Variables with VAEs despite Training Bias/beta=1, mu=[2] training curves.png
diff --git a/...vering Latent Variables with VAEs despite Training Bias/example test images.png b/...vering Latent Variables with VAEs despite Training Bias/example test images.png
diff --git a/...ng Latent Variables with VAEs despite Training Bias/example training noised.png b/...ng Latent Variables with VAEs despite Training Bias/example training noised.png
diff --git a/...ecovering Latent Variables with VAEs despite Training Bias/example training.png b/...ecovering Latent Variables with VAEs despite Training Bias/example training.png
diff --git a/...ing Latent Variables with VAEs despite Training Bias/mu=[0] training curves.png b/...ing Latent Variables with VAEs despite Training Bias/mu=[0] training curves.png
diff --git a/bin/dockerhub_run.sh b/bin/dockerhub_run.sh
@@ -2,6 +2,6 @@ FILE=Gemfile.lock
 if [ -f "$FILE" ]; then
     rm $FILE
 fi
-docker run --rm -v "$PWD:/srv/jekyll/" -p "8080:8080" \
+docker run --rm -v "$PWD:/srv/jekyll/" -p "8090:8090" \
                     -it amirpourmand/al-folio:v0.7.0 bundler  \
-                    exec jekyll serve --watch --port=8080 --host=0.0.0.0 
+                    exec jekyll serve --watch --port=8090 --host=0.0.0.0