CrossEntropyLoss with weights
I need weights in CrossEntropyLoss (actually multiple, but the same issue).  The documentation talks about tensors copied from other tensors, but there is no tensor to copy from in the init.  So I'm stuck.
To make the weights unquestionably simple, I use ones.
class JJG_Transformer(pl.LightningModule):

    def __init__(self, alphanet_plus_2, letter_weights_per_position):
        super(JJG_Transformer, self).__init__()
        self.criterions = []
        for weight in self.letter_weights_per_position:
            weight = torch.ones((94))
            self.criterions.append( torch.nn.CrossEntropyLoss(weight=weight) )
    def validation_step(self, batch, batch_idx):
        batch_im, batch_true_value_NT, batch_letter_transformer_input = batch
        out_NTA = self(batch_im, batch_letter_transformer_input)
        loss0 = self.criterions[0](out_NTA[:,0,:], batch_true_value_NT[:,0])
        loss1 = self.criterions[1](out_NTA[:,1,:], batch_true_value_NT[:,1])
        loss = loss0 + loss1
        tensorboard_logs = {'val_loss': loss, 'val_loss0': loss0, 'val_loss1':loss1}
        return {'val_loss': loss, 'log': tensorboard_logs}


File "/home/john/Documents/GitHub/Offline_Handwriting_Recognition/Solutions/Aug2020_simple_transformer/src/kiss_transformer.py", line 254, in <module>
  trainer.fit(model, train_dataloader=train_loader, val_dataloaders=val_loader)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/states.py", line 34, in wrapped_fn
  result = fn(self, *args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1017, in fit
  self.accelerator_backend.train(model)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/accelerators/ddp_backend.py", line 56, in train
  self.ddp_train(process_idx=self.task_idx, mp_queue=None, model=model)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/accelerators/ddp_backend.py", line 219, in ddp_train
  results = self.trainer.run_pretrain_routine(model)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1196, in run_pretrain_routine
  self._run_sanity_check(ref_model, model)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1229, in _run_sanity_check
  eval_results = self._evaluate(model, self.val_dataloaders, max_batches, False)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 325, in _evaluate
  output = self.evaluation_forward(model, batch, batch_idx, dataloader_idx, test_mode)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 609, in evaluation_forward
  output = model(*args)
File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
  result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/overrides/data_parallel.py", line 160, in forward
  output = self.module.validation_step(*inputs[0], **kwargs[0])
File "/home/john/Documents/GitHub/Offline_Handwriting_Recognition/Solutions/Aug2020_simple_transformer/src/kiss_transformer.py", line 128, in validation_step
  loss0 = self.criterions[0](out_NTA[:,0,:], batch_true_value_NT[:,0])
File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
  result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/loss.py", line 948, in forward
  ignore_index=self.ignore_index, reduction=self.reduction)
File "/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py", line 2422, in cross_entropy
  return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
File "/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py", line 2218, in nll_loss
  ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
RuntimeError: Expected object of device type cuda but got device type cpu for argument #3 'weight' in call to _thnn_nll_loss_forward
Traceback (most recent call last):
File "kiss_transformer.py", line 254, in <module>
  trainer.fit(model, train_dataloader=train_loader, val_dataloaders=val_loader)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/states.py", line 34, in wrapped_fn
  result = fn(self, *args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1030, in fit
  results = self.accelerator_backend.spawn_ddp_children(model)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/accelerators/ddp_backend.py", line 118, in spawn_ddp_children
  results = self.ddp_train(local_rank, mp_queue=None, model=model, is_master=True)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/accelerators/ddp_backend.py", line 219, in ddp_train
  results = self.trainer.run_pretrain_routine(model)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1196, in run_pretrain_routine
  self._run_sanity_check(ref_model, model)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1229, in _run_sanity_check
  eval_results = self._evaluate(model, self.val_dataloaders, max_batches, False)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 325, in _evaluate
  output = self.evaluation_forward(model, batch, batch_idx, dataloader_idx, test_mode)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 609, in evaluation_forward
  output = model(*args)
File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
  result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/overrides/data_parallel.py", line 160, in forward
  output = self.module.validation_step(*inputs[0], **kwargs[0])
File "kiss_transformer.py", line 128, in validation_step
  loss0 = self.criterions[0](out_NTA[:,0,:], batch_true_value_NT[:,0])
File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
  result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/loss.py", line 948, in forward
  ignore_index=self.ignore_index, reduction=self.reduction)
File "/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py", line 2422, in cross_entropy
  return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
File "/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py", line 2218, in nll_loss
  ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
RuntimeError: Expected object of device type cuda but got device type cpu for argument #3 'weight' in call to _thnn_nll_loss_forward

trainer = pl.Trainer( gpus=[0, 1],  
                accumulate_grad_batches=16, 
                max_epochs=500, 
                check_val_every_n_epoch=1, 
                distributed_backend='ddp', 

pl__version__ 0.9.0rc12