EarlyStop and Model Checkpoint do not train or log anythings.

[AlexPapas]

Describe the bug
As per documentation Early stopping and Model checkpoint can take box recall

But when used there is no training happening, the training always stops on the number of epochs set by the Early stopping. If removed, trainings is taking place.
In addtion, when I use callbacks, not metrics are been recorded with MLFLow. when removed, everything works.

Can you please advice?

from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping

#initial the model and change the corresponding config file
m = main.deepforest(label_dict={"Tree": 0},
                    transforms=get_transform
                    )
# load the latest model
m.use_release()

# Train
m.config["train"]["csv_file"] = train_file
m.config["train"]["root_dir"] = os.path.dirname(train_file)
m.config["score_thresh"] = 0.4
m.config['nms_thersh'] = 0.05
m.config["train"]['epochs'] = 80
m.config["workers"] = 12
m.config["batch_size"] = 8

# Validation
m.config["validation"]["csv_file"] = validation_file
m.config["validation"]["root_dir"] = os.path.dirname(validation_file)
m.config["validation"]["val_accuracy_interval"] = 10 

m.config["train"]["scheduler"] = {'type': None,
 'params': {'T_max': 10,
  'eta_min': 1e-05,
  'lr_lambda': 'lambda epoch: 0.95 ** epoch',
  'step_size': 30,
  'gamma': 0.1,
  'milestones': [50, 100],
  'mode': 'min',
  'factor': 0.1,
  'patience': 10,
  'threshold': 0.0001,
  'threshold_mode': 'rel',
  'cooldown': 0,
  'min_lr': 0,
  'eps': 1e-08}}

checkpoint_callback = ModelCheckpoint(
    dirpath=os.path.dirname(train_file) +"/models",
    monitor='box_recall', 
    mode="max",
    save_top_k=1,
    filename="box_recall-{epoch:02d}-{box_recall:.2f}")

early_stop_callback = EarlyStopping(monitor='box_recall',
                                patience=6)

m.create_trainer(logger=True,
                 enable_checkpointing=True,
                 log_every_n_steps=3,
                 default_root_dir=os.path.dirname(train_file) +"/models",
                 enable_progress_bar =True,
                 enable_model_summary =True,
    callbacks=[ checkpoint_callback, early_stop_callback]
    )

start_time =time.time()

mlflow.set_experiment(experiment_id="3892779529544240")
mlflow.pytorch.autolog(
    disable=True,
    disable_for_unsupported_versions=False,
    silent=True)

with mlflow.start_run(run_name=f'tree_detection_deep') as run:
    m.trainer.fit(m)

print(f'--- Trainings on GPU: {(time.time()- start_time):.2f} seconds ----')