def toggle_optimizer(self, optimizer: Optimizer, optimizer_idx: int):          Override for your own behavior +        It works with ``untoggle_optimizer`` to make sure param_requires_grad_state is properly reset.+         Args:-            optimizer:-            optimizer_idx:+            optimizer: Current optimizer used in training_loop+            optimizer_idx: Current optimizer idx in training_loop         """-        for param in self.parameters():-            param.requires_grad = False+        param_requires_grad_state = {}+        # make sure current optimizer is latest to be iterated over.+        optimizers = [opt for opt in self.optimizers(use_pl_optimizer=False) if opt != optimizer] + [optimizer]+        num_optimizers = len(optimizers) - 1+        for opt_idx, opt in enumerate(optimizers):+            for group in opt.param_groups:+                for param in group['params']:+                    if num_optimizers == opt_idx:

 def toggle_optimizer(self, optimizer: Optimizer, optimizer_idx: int): 

 def __test_given_model(self, model, test_dataloaders):          return results +    def predict(+        self,+        model: Optional[LightningModule] = None,+        dataloaders: Optional[Union[DataLoader, List[DataLoader]]] = None,+    ):+        r"""++        Separates from fit to make sure you never run on your predictions set until you want to.++        This will call the model forward function to compute predictions.++        Args:+            model: The model to predict on.++            dataloaders: Either a single+                Pytorch Dataloader or a list of them, specifying inference samples.++        Returns:+            The final test result dictionary. If no test_epoch_end is defined returns a list of dictionaries+        """++        # --------------------+        # SETUP HOOK+        # --------------------+        if not (+            isinstance(dataloaders, DataLoader)+            or isinstance(dataloaders, (list, tuple))+            and all(isinstance(d, DataLoader) for d in dataloaders)+        ):+            raise MisconfigurationException(+                'You need to pass a dataloader or a list of dataloaders to `trainer.predict`. '+            )++        if model is None:+            raise MisconfigurationException(+                'You need to pass a model to `trainer.predict`. '+            )++        # attach data+        if dataloaders is not None:+            self.data_connector.attach_dataloaders(model, test_dataloaders=dataloaders)++        # set path variable+        self.is_predicting = True+        os.environ['PL_TESTING_MODE'] = '1'+        self.model = model++        results = self.fit(model)

 def __test_given_model(self, model, test_dataloaders):          return results +    def predict(+        self,+        model: Optional[LightningModule] = None,+        dataloaders: Optional[Union[DataLoader, List[DataLoader]]] = None,+    ):+        r"""++        Separates from fit to make sure you never run on your predictions set until you want to.++        This will call the model forward function to compute predictions.++        Args:+            model: The model to predict on.++            dataloaders: Either a single+                Pytorch Dataloader or a list of them, specifying inference samples.++        Returns:+            The final test result dictionary. If no test_epoch_end is defined returns a list of dictionaries+        """++        # --------------------+        # SETUP HOOK+        # --------------------+        if not (+            isinstance(dataloaders, DataLoader)+            or isinstance(dataloaders, (list, tuple))+            and all(isinstance(d, DataLoader) for d in dataloaders)+        ):+            raise MisconfigurationException(+                'You need to pass a dataloader or a list of dataloaders to `trainer.predict`. '+            )++        if model is None:+            raise MisconfigurationException(+                'You need to pass a model to `trainer.predict`. '+            )++        # attach data+        if dataloaders is not None:+            self.data_connector.attach_dataloaders(model, test_dataloaders=dataloaders)++        # set path variable+        self.is_predicting = True

 def __init__(         self._device_type = DeviceType.CPU         self._distrib_type = None         self._running_stage = None+        self.is_predicting = False

 # See the License for the specific language governing permissions and # limitations under the License. from collections import defaultdict-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, List, Optional  import torch  from pytorch_lightning.core.step_result import Result+from pytorch_lightning.trainer.states import RunningStage from pytorch_lightning.utilities import DistributedType, LightningEnum  -class LoggerStages(LightningEnum):

 def validation_step(self, args):     def test_step(self, args):         return self._step(args) +    def forward(self, args):+        return self._step(args)+

 def test_trainer_profiler_incorrect_arg_type(profiler):                        match=r"Only None, bool, str and subclasses of `BaseProfiler`"                              r" are valid values for `Trainer`'s `profiler` parameter. *"):         Trainer(profiler=profiler)+++def test_pytorch_profiler(tmpdir):+    class TestModel(BoringModel):+        def training_step(self, batch, batch_idx):+            output = self.layer(batch)+            loss = self.loss(batch, output)+            return {"loss": loss}++    model = TestModel()

 def __del__(self):         """Close profiler's stream."""         if self.output_file:             self.output_file.close()+++class PytorchProfiler(BaseProfiler):+    """+    This profiler uses PyTorch's Autograd Profiler and let's you inspect the cost of

 def test_trainer_profiler_incorrect_arg_type(profiler):                        match=r"Only None, bool, str and subclasses of `BaseProfiler`"                              r" are valid values for `Trainer`'s `profiler` parameter. *"):         Trainer(profiler=profiler)+++def test_pytorch_profiler(tmpdir):+    class TestModel(BoringModel):+        def training_step(self, batch, batch_idx):+            output = self.layer(batch)+            loss = self.loss(batch, output)+            return {"loss": loss}++    model = TestModel()++    limit_train_batches = 2+    trainer = Trainer(+        default_root_dir=tmpdir,+        limit_train_batches=limit_train_batches,+        limit_val_batches=2,+        max_epochs=1,+        profiler='pytorch'+    )++    trainer.fit(model)

 def validation_step(self, batch, batch_idx):         weights_summary=None,         accelerator="ddp",         gpus=2,+        profiler="pytorch"

 def custom_processing_step(self, data):     model = MyModel(profiler)     trainer = Trainer(profiler=profiler, max_epochs=1) ++PyTorch Profiling+--------------------++Autograd includes a profiler that lets you inspect the cost of different operators+inside your model - both on the CPU and GPU.++.. code-block:: python++    trainer = Trainer(..., profiler="pytorch")++    or++    profiler = PytorchProfiler(+            output_filename = ...+            enabled = ...+            use_cuda = ...+            record_shapes = ...+            profile_memory = ...+            with_stack = ...+            use_kineto = ...+            use_cpu = ...+            emit_nvtx = ...+            export_to_chrome = ...+            sort_by_key = ...+            path_to_export_trace = ...+    )+    trainer = Trainer(..., profiler=profiler)++The profiler's results will be printed at the completion of a training `fit()`. This profiler+report can be quite long, so you can also specify an `output_filename` to save the report instead+of logging it to the output in your terminal. The output below shows the profiling for the action+`get_train_batch`.+This profiler will record only for `training_step`, `evaluation_step` and `test_step` functions.++.. code-block:: python++    Profiler Report++    Profile stats for: training_step+    ---------------------  ---------------  ---------------  ---------------  ---------------  ---------------+    Name                   Self CPU total %  Self CPU total   CPU total %      CPU total        CPU time avg+    ---------------------  ---------------  ---------------  ---------------  ---------------  ---------------+    t                      62.10%           1.044ms          62.77%           1.055ms          1.055ms+    addmm                  32.32%           543.135us        32.69%           549.362us        549.362us+    mse_loss               1.35%            22.657us         3.58%            60.105us         60.105us+    mean                   0.22%            3.694us          2.05%            34.523us         34.523us+    div_                   0.64%            10.756us         1.90%            32.001us         16.000us+    ones_like              0.21%            3.461us          0.81%            13.669us         13.669us+    sum_out                0.45%            7.638us          0.74%            12.432us         12.432us+    transpose              0.23%            3.786us          0.68%            11.393us         11.393us+    as_strided             0.60%            10.060us         0.60%            10.060us         3.353us+    to                     0.18%            3.059us          0.44%            7.464us          7.464us+    empty_like             0.14%            2.387us          0.41%            6.859us          6.859us+    empty_strided          0.38%            6.351us          0.38%            6.351us          3.175us+    fill_                  0.28%            4.782us          0.33%            5.566us          2.783us+    expand                 0.20%            3.336us          0.28%            4.743us          4.743us+    empty                  0.27%            4.456us          0.27%            4.456us          2.228us+    copy_                  0.15%            2.526us          0.15%            2.526us          2.526us+    broadcast_tensors      0.15%            2.492us          0.15%            2.492us          2.492us+    size                   0.06%            0.967us          0.06%            0.967us          0.484us+    is_complex             0.06%            0.961us          0.06%            0.961us          0.481us+    stride                 0.03%            0.517us          0.03%            0.517us          0.517us+    ---------------------  ---------------  ---------------  ---------------  ---------------  ---------------+    Self CPU time total: 1.681ms++When running with `PytorchProfiler(emit_nvtx=True)`. You should run as following:++nvprof --profile-from-start off -o trace_name.prof -- <regular command here>++To visualize the profiled operation, you can either:++* Use: nvvp trace_name.prof++* Use: python -c 'import torch; print(torch.autograd.profiler.load_nvprof("trace_name.prof"))'+ """ -from pytorch_lightning.profiler.profilers import AdvancedProfiler, BaseProfiler, PassThroughProfiler, SimpleProfiler+from pytorch_lightning.profiler.profilers import (+    AdvancedProfiler,+    BaseProfiler,+    PassThroughProfiler,+    PytorchProfiler,+    SimpleProfiler,+)  __all__ = [     'BaseProfiler',     'SimpleProfiler',     'AdvancedProfiler',     'PassThroughProfiler',+    "PytorchProfiler",

 def custom_processing_step(self, data):     model = MyModel(profiler)     trainer = Trainer(profiler=profiler, max_epochs=1) ++PyTorch Profiling+--------------------++Autograd includes a profiler that lets you inspect the cost of different operators+inside your model - both on the CPU and GPU.++.. code-block:: python++    trainer = Trainer(..., profiler="pytorch")++    or++    profiler = PytorchProfiler(+            output_filename = ...+            enabled = ...+            use_cuda = ...+            record_shapes = ...+            profile_memory = ...+            with_stack = ...+            use_kineto = ...+            use_cpu = ...+            emit_nvtx = ...+            export_to_chrome = ...+            sort_by_key = ...+            path_to_export_trace = ...+    )+    trainer = Trainer(..., profiler=profiler)++The profiler's results will be printed at the completion of a training `fit()`. This profiler

 def custom_processing_step(self, data):     model = MyModel(profiler)     trainer = Trainer(profiler=profiler, max_epochs=1) ++PyTorch Profiling+--------------------++Autograd includes a profiler that lets you inspect the cost of different operators+inside your model - both on the CPU and GPU.++.. code-block:: python++    trainer = Trainer(..., profiler="pytorch")++    or++    profiler = PytorchProfiler(+            output_filename = ...+            enabled = ...+            use_cuda = ...+            record_shapes = ...+            profile_memory = ...+            with_stack = ...+            use_kineto = ...+            use_cpu = ...+            emit_nvtx = ...+            export_to_chrome = ...+            sort_by_key = ...+            path_to_export_trace = ...+    )+    trainer = Trainer(..., profiler=profiler)

 def custom_processing_step(self, data):     model = MyModel(profiler)     trainer = Trainer(profiler=profiler, max_epochs=1) ++PyTorch Profiling+--------------------++Autograd includes a profiler that lets you inspect the cost of different operators+inside your model - both on the CPU and GPU.++.. code-block:: python++    trainer = Trainer(..., profiler="pytorch")++    or++    profiler = PytorchProfiler(+            output_filename = ...+            enabled = ...+            use_cuda = ...+            record_shapes = ...+            profile_memory = ...+            with_stack = ...+            use_kineto = ...+            use_cpu = ...+            emit_nvtx = ...+            export_to_chrome = ...+            sort_by_key = ...+            path_to_export_trace = ...+    )+    trainer = Trainer(..., profiler=profiler)++The profiler's results will be printed at the completion of a training `fit()`. This profiler+report can be quite long, so you can also specify an `output_filename` to save the report instead+of logging it to the output in your terminal. The output below shows the profiling for the action+`get_train_batch`.+This profiler will record only for `training_step`, `evaluation_step` and `test_step` functions.

 def custom_processing_step(self, data):     model = MyModel(profiler)     trainer = Trainer(profiler=profiler, max_epochs=1) ++PyTorch Profiling+--------------------++Autograd includes a profiler that lets you inspect the cost of different operators+inside your model - both on the CPU and GPU.++.. code-block:: python++    trainer = Trainer(..., profiler="pytorch")++    or++    profiler = PytorchProfiler(+            output_filename = ...+            enabled = ...+            use_cuda = ...+            record_shapes = ...+            profile_memory = ...+            with_stack = ...+            use_kineto = ...+            use_cpu = ...+            emit_nvtx = ...+            export_to_chrome = ...+            sort_by_key = ...+            path_to_export_trace = ...+    )+    trainer = Trainer(..., profiler=profiler)++The profiler's results will be printed at the completion of a training `fit()`. This profiler+report can be quite long, so you can also specify an `output_filename` to save the report instead+of logging it to the output in your terminal. The output below shows the profiling for the action+`get_train_batch`.+This profiler will record only for `training_step`, `evaluation_step` and `test_step` functions.++.. code-block:: python

 def validation_step(self, args):     def test_step(self, args):         return self._step(args) +    def forward(self, args):+        return self._step(args)+

 def toggle_optimizer(self, optimizer: Optimizer, optimizer_idx: int):          Override for your own behavior +        It works with ``untoggle_optimizer`` to make sure param_requires_grad_state is properly reset.+         Args:-            optimizer:-            optimizer_idx:+            optimizer: Current optimizer used in training_loop+            optimizer_idx: Current optimizer idx in training_loop         """-        for param in self.parameters():-            param.requires_grad = False+        param_requires_grad_state = {}+        # make sure current optimizer is latest to be iterated over.+        optimizers = [opt for opt in self.optimizers(use_pl_optimizer=False) if opt != optimizer] + [optimizer]+        num_optimizers = len(optimizers) - 1+        for opt_idx, opt in enumerate(optimizers):+            for group in opt.param_groups:+                for param in group['params']:+                    if num_optimizers == opt_idx:

 def __test_given_model(self, model, test_dataloaders):          return results +    def predict(+        self,+        model: Optional[LightningModule] = None,+        dataloaders: Optional[Union[DataLoader, List[DataLoader]]] = None,+    ):+        r"""++        Separates from fit to make sure you never run on your predictions set until you want to.++        This will call the model forward function to compute predictions.++        Args:+            model: The model to predict on.++            dataloaders: Either a single+                Pytorch Dataloader or a list of them, specifying inference samples.++        Returns:+            The final test result dictionary. If no test_epoch_end is defined returns a list of dictionaries+        """++        # --------------------+        # SETUP HOOK+        # --------------------+        if not (+            isinstance(dataloaders, DataLoader)+            or isinstance(dataloaders, (list, tuple))+            and all(isinstance(d, DataLoader) for d in dataloaders)+        ):+            raise MisconfigurationException(+                'You need to pass a dataloader or a list of dataloaders to `trainer.predict`. '+            )++        if model is None:+            raise MisconfigurationException(+                'You need to pass a model to `trainer.predict`. '+            )++        # attach data+        if dataloaders is not None:+            self.data_connector.attach_dataloaders(model, test_dataloaders=dataloaders)++        # set path variable+        self.is_predicting = True

 # See the License for the specific language governing permissions and # limitations under the License. from collections import defaultdict-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, List, Optional  import torch  from pytorch_lightning.core.step_result import Result+from pytorch_lightning.trainer.states import RunningStage from pytorch_lightning.utilities import DistributedType, LightningEnum  -class LoggerStages(LightningEnum):

 def testing(self, val: bool) -> None:         if val:             self._running_stage = RunningStage.TESTING         else:-            self._running_stage = None+            if self._running_stage == RunningStage.TRAINING:+                pass+            else:+                self._running_stage = None

 def update_logger_connector(self) -> None:         callback_metrics = {}         batch_pbar_metrics = {}         batch_log_metrics = {}-        is_train = self._stage in LoggerStages.TRAIN.value+        is_train = self._stage in RunningStage.TRAINING

 def validation_step(self, args):     def test_step(self, args):         return self._step(args) +    def forward(self, args):+        return self._step(args)+

 def __init__(         self._device_type = DeviceType.CPU         self._distrib_type = None         self._running_stage = None+        self.is_predicting = False