Multi-GPU training. learning rate is all zero in tensorboard .
🐛 Bug
I used LearningrateLogger to log learning rate. But in tensorboard learning rate is all zero.

To Reproduce
Steps to reproduce the behavior:

install 0.10.0rc1
set gpus in Trainer bigger than 1
use lr_logger = LearningRateLogger(logging_interval='step') to log learning rate

Code sample
pseudocode
class BartFineTuner(pl.LightningModule):
    def __init__(self, hparams, learning_rate=None):
        super(BartFineTuner, self).__init__()
        self.hparams = hparams
        self.learning_rate = learning_rate

        model_name = hparams.model_name_or_path
        self.model, self.tokenizer = load_model_and_tokenizer(model_name, config_dict)

        self.loss = nn.CrossEntropyLoss(ignore_index=self.tokenizer.pad_token_id)

    def is_logger(self):
        return self.trainer.global_rank <= 0

    def forward(
            self, input_ids, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, lm_labels=None,
            use_cache=False

    ):
        return self.model(
            input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            decoder_attention_mask=decoder_attention_mask,
            use_cache=use_cache
        )

    def _step(self, batch):
        tgt_ids = batch["target_ids"]
        decoder_input_ids = shift_tokens_right(tgt_ids, self.tokenizer.pad_token_id)

        outputs = self(
            input_ids=batch["source_ids"],
            attention_mask=batch["source_mask"],
            decoder_input_ids=decoder_input_ids,
            decoder_attention_mask=None,  
            use_cache=False
        )
        target = batch["target_ids"]
        if self.hparams.epsilon > 0:
            output_softmax = F.log_softmax(outputs[0], dim=-1)
            loss, _ = label_smoothed_nll_loss(output_softmax, target, self.hparams.epsilon,
                                              ignore_index=self.tokenizer.pad_token_id)
        else:
            ce_loss_fct = torch.nn.CrossEntropyLoss(ignore_index=self.tokenizer.pad_token_id)
            loss = ce_loss_fct(outputs[0].view(-1, outputs[0].shape[-1]), target.view(-1))
        return loss

    def training_step(self, batch, batch_idx):
        loss = self._step(batch)
        self.log('train_loss', loss)
        return {"loss": loss}

    def training_epoch_end(self, outputs):
        avg_train_loss = torch.stack([x["loss"] for x in outputs]).mean()
        self.log("avg_train_loss", avg_train_loss)

    def validation_step(self, batch, batch_idx):
        loss = self._step(batch)
        return {"val_loss": loss}

    def validation_epoch_end(self, outputs):
        avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean()
        self.log("val_loss", avg_loss)
        return {"avg_val_loss": avg_loss}

    def total_steps(self) -> int:
        """The number of total training steps that will be run. Used for lr scheduler purposes."""
        num_devices = max(1, self.hparams.n_gpu)  
        effective_batch_size = self.hparams.train_batch_size * self.hparams.gradient_accumulation_steps * num_devices

        dataset_size = len(self.train_loader.dataset)
        return (dataset_size / effective_batch_size) * self.hparams.num_train_epochs

    def setup(self, mode):
        if mode == "fit":
            self.train_loader = self.get_dataloader()

    def configure_optimizers(self):

        model = self.model
        no_decay = ["bias", "LayerNorm.weight"]
        optimizer_grouped_parameters = [
            {
                "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
                "weight_decay": self.hparams.weight_decay,
            },
            {
                "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
                "weight_decay": 0.0,
            },
        ]

        lr = self.hparams.learning_rate

        optimizer = AdamW(optimizer_grouped_parameters, lr=lr, eps=self.hparams.adam_epsilon)
        self.opt = optimizer
        scheduler = get_polynomial_decay_schedule_with_warmup(
            self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=self.total_steps()
        )
        return [optimizer], [scheduler]


    def get_dataloader(self):
        train_dataset = get_dataset(tokenizer=self.tokenizer, args=self.hparams, type_path="train",
                                    train_type=self.hparams.train_type)
        dataloader = DataLoader(train_dataset, batch_size=self.hparams.train_batch_size, drop_last=True, shuffle=True,
                                num_workers=1)
        return dataloader
tb_logger = pl_loggers.TensorBoardLogger(os.path.join("logs", self.args["name"], args.train_type))
lr_logger = LearningRateLogger(logging_interval='step')
train_params = dict(
            gpus=2,
            early_stop_callback=False,
            precision=16 if args.fp_16 else 32,
            amp_level='O1',
            checkpoint_callback=False,
            callbacks=[lr_logger],
            logger=tb_logger,
        )
model = BartFineTuner(args)
trainer = pl.Trainer(**train_params)
trainer.fit(model)
Expected behavior
Log real learning rate
Environment

CUDA:
- GPU:
- Tesla P4
- Tesla P4
- available:         True
- version:           10.2
Packages:
- numpy:             1.18.5
- pyTorch_debug:     False
- pyTorch_version:   1.6.0
- pytorch-lightning: 0.10.0rc1
- tqdm:              4.50.0
System:
- OS:                Linux
- architecture:
- 64bit
-
- processor:         x86_64
- python:            3.7.9
- version:           #32~16.04.2-Ubuntu SMP Thu Jul 20 10:19:48 UTC 2017

Additional context