train_small.cfg

# Model/Embeddings
word_vec_size: 100  # Word embedding size for src and tgt
share_embeddings: True  # Share embeddings from src and tgt

# Model/Embedding Features
feat_vec_size: -1  # Attribute embedding size. -1 means <nb_features>**0.7
feat_merge: mlp  # Merge action for incorporating feature embeddings [concat|sum|mlp]


# Model Structure
model_type: text  # Type of source model to use [text|img|audio]
model_dtype: fp32
encoder_type: brnn  # Type of encoder [rnn|brnn|transformer|cnn]
decoder_type: rnn  # Type of decoder [rnn|transformer|cnn]
param_init: 0.1  # Uniform distribution with support (-param_init, +param_init)

#layers: 2
#nb_branches: 3
rnn_size: 100
input_feed: 1
bridge: True
rnn_type: LSTM


# Model/Attention
global_attention: general  # Type of attn to use [dot|general|mlp|none]
global_attention_function: softmax  # [softmax|sparsemax]
generator_function: softmax

# Model/Copy
copy_attn: True
reuse_copy_attn: True  # Reuse standard attention for copy
copy_attn_force: True  # When available, train to copy


#  Files and logs
data: experiments/wikibio/small/data/data  # path to datafile from preprocess.py
#weights_file: data/wikibio/train_weights_reg2.txt
save_model: experiments/wikibio/small/models/model  # path to store checkpoints
log_file: experiments/wikibio/small/train-log.txt

report_every: 10  # log current loss every X steps
save_checkpoint_steps: 2500  # save a cp every X steps


# Gpu related:
gpu_ranks: [0]  # ids of gpus to use
world_size: 1  # total number of distributed processes
gpu_backend: nccl  # type of torch distributed backend
gpu_verbose_level: 0
master_ip: localhost
master_port: 10000
seed: 123


# Optimization & training
batch_size: 64
batch_type: sents
normalization: sents
accum_count: [1]  # Update weights every X batches
accum_steps: [0]  # steps at which accum counts value changes
valid_steps: 100000  # run models on validation set every X steps
train_steps: 100000
optim: adam
max_grad_norm: 5
dropout: .3
adam_beta1: 0.9
adam_beta2: 0.999
label_smoothing: 0.0
average_decay: 0
average_every: 1

# RL training (comment to train without RL)
train_with_rl: True
rl_gamma_loss: .5
rl_metric: hss
ref_path: /local/rebuffel/dtt-hallucinations-disentanglement/data/wikibio/small/train_output.txt
max_generator_batches: 0  # set shard_size to zero explictly

# Learning rate
learning_rate: 0.001
learning_rate_decay: 0.5  # lr *= lr_decay
start_decay_step: 5000
decay_steps: 10000