Edge case bug when validating on multiple data sets if .log is not called at least once for each data set.
🐛 Bug
When validating on multiple data sets, if you call self.log(..) for any data set, self.log(..) also has to have been called for all previous data sets at least once. If not, you get a KeyError when the results are auto reduced. I encountered this because for one of my validation sets I only created and logged images.
Reproduce
import pytorch_lightning as pl
import torch
from torch.utils.data import DataLoader, TensorDataset


class Module(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.model = torch.nn.Linear(1, 1)

    def validation_step(self, batch, batch_idx, dataset_idx):
        if dataset_idx == 0:
            # When you uncomment the following line, everything works.
            # self.log("test1", 0.)
            # Just calling self.log for dataset 0 and not for dataset 1 also works
            pass
        else:
            self.log("test2", 0.)

    def val_dataloader(self):
        return (
            DataLoader(TensorDataset(torch.ones(2, 1))),
            DataLoader(TensorDataset(torch.ones(2, 1)))
        )

    def training_step(self, batch, batch_idx):
        return torch.mean(self.model(batch[0]))

    def train_dataloader(self):
        return DataLoader(TensorDataset(torch.ones(2, 1)))

    def configure_optimizers(self):
        return torch.optim.SGD(self.parameters(), lr=0.01)


if __name__ == "__main__":
    trainer = pl.Trainer()
    trainer.fit(Module())
Error you get:
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/trainer.py", line 690, in run_sanity_check
    _, eval_results = self.run_evaluation(test_mode=False, max_batches=self.num_sanity_val_batches)
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/trainer.py", line 622, in run_evaluation
    deprecated_eval_results = self.evaluation_loop.evaluation_epoch_end()
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/evaluation_loop.py", line 203, in evaluation_epoch_end
    self.trainer.logger_connector.evaluation_epoch_end(self.testing)
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py", line 225, in evaluation_epoch_end
    self.cached_results.has_batch_loop_finished = True
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/connectors/logger_connector/epoch_result_store.py", line 441, in has_batch_loop_finished
    self.auto_reduce_results_on_epoch_end()
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/connectors/logger_connector/epoch_result_store.py", line 431, in auto_reduce_results_on_epoch_end
    hook_result.auto_reduce_results_on_epoch_end()
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/connectors/logger_connector/epoch_result_store.py", line 195, in auto_reduce_results_on_epoch_end
    epoch_metrics = self._internals[dl_idx]
KeyError: 0

Environment
* CUDA:
        - GPU:
        - available:         False
        - version:           10.2
* Packages:
        - numpy:             1.19.4
        - pyTorch_debug:     False
        - pyTorch_version:   1.7.1
        - pytorch-lightning: 1.1.0
        - tqdm:              4.54.1
* System:
        - OS:                Linux
        - architecture:
                - 64bit
                - ELF
        - processor:         x86_64
        - python:            3.6.9
        - version:           #62-Ubuntu SMP Mon Nov 23 19:20:19 UTC 2020