make pytorch=1.1 and cuda 10, should work

Author: frankxu2004

components/vocab.py

@@ -1,10 +1,10 @@
 # coding=utf-8
 
 from __future__ import print_function
-import argparse
+
 from collections import Counter
 from itertools import chain
-import torch
+
 
 class VocabEntry(object):
     def __init__(self):
@@ -46,6 +46,11 @@ def add(self, word):
     def is_unk(self, word):
         return word not in self
 
+    def merge(self, other_vocab_entry):
+        for word in other_vocab_entry.word2id:
+            self.add(word)
+
+
     @staticmethod
     def from_corpus(corpus, size, freq_cutoff=0):
         vocab_entry = VocabEntry()
@@ -55,7 +60,8 @@ def from_corpus(corpus, size, freq_cutoff=0):
         singletons = [w for w in word_freq if word_freq[w] == 1]
         print('number of word types: %d, number of word types w/ frequency > 1: %d' % (len(word_freq),
                                                                                        len(non_singletons)))
-        print('singletons: %s' % singletons)
+        print('number of singletons: ', len(singletons))
+        # print('singletons: %s' % singletons)
 
         top_k_words = sorted(word_freq.keys(), reverse=True, key=word_freq.get)[:size]
         words_not_included = []

config/env/frank.yml

@@ -1,17 +1,14 @@
 name: tranX
 channels:
+  - pytorch
   - anaconda
   - defaults
 dependencies:
-  - cudatoolkit=10.1.168=0
-  - cudnn=7.6.0=cuda10.1_0
-  - cupti=10.1.168
+  - astor=0.7.1
+  - cudatoolkit=10.0.130
   - python=3.7.3
+  - pytorch=1.1.0
   - pip:
-    - compare-mt==0.2.7
-    - elasticsearch==7.0.5
-    - six==1.12.0
-    - xgboost==0.90
-    - torch==1.0.1.post2
-    - astor==0.7.1
+    - six
+    - xgboost
     - tqdm

datasets/conala/dataset.py

@@ -15,6 +15,7 @@
 from datasets.conala.evaluator import ConalaEvaluator
 from datasets.conala.util import *
 
+assert astor.__version__ == '0.7.1'
 
 def preprocess_conala_dataset(train_file, test_file, grammar_file, src_freq=3, code_freq=3,
                               mined_data_file=None, vocab_size=20000, num_mined=0, out_dir='data/conala'):
@@ -33,11 +34,23 @@ def preprocess_conala_dataset(train_file, test_file, grammar_file, src_freq=3, c
     dev_examples = train_examples[:200]
     train_examples = train_examples[200:]
 
+    mined_examples = None
     if mined_data_file and num_mined > 0:
         print("use mined data: ", num_mined)
         print("from file: ", mined_data_file)
         mined_examples = preprocess_dataset(mined_data_file, name='mined', transition_system=transition_system,
                                             firstk=num_mined)
+        # mined_src_vocab = VocabEntry.from_corpus([e.src_sent for e in train_examples], size=vocab_size,
+        #                                    freq_cutoff=src_freq)
+        # mined_primitive_tokens = [map(lambda a: a.action.token,
+        #                         filter(lambda a: isinstance(a.action, GenTokenAction), e.tgt_actions))
+        #                     for e in train_examples]
+        # mined_primitive_vocab = VocabEntry.from_corpus(mined_primitive_tokens, size=vocab_size, freq_cutoff=code_freq)
+        #
+        # # generate vocabulary for the code tokens!
+        # mined_code_tokens = [transition_system.tokenize_code(e.tgt_code, mode='decoder') for e in train_examples]
+        # mined_code_vocab = VocabEntry.from_corpus(mined_code_tokens, size=vocab_size, freq_cutoff=code_freq)
+
         pickle.dump(mined_examples, open(os.path.join(out_dir, 'pre_{}.bin'.format(num_mined)), 'wb'))
         train_examples += mined_examples
 
@@ -57,6 +70,7 @@ def preprocess_conala_dataset(train_file, test_file, grammar_file, src_freq=3, c
 
     # generate vocabulary for the code tokens!
     code_tokens = [transition_system.tokenize_code(e.tgt_code, mode='decoder') for e in train_examples]
+
     code_vocab = VocabEntry.from_corpus(code_tokens, size=vocab_size, freq_cutoff=code_freq)
 
     vocab = Vocab(source=src_vocab, primitive=primitive_vocab, code=code_vocab)

exp.py

@@ -3,6 +3,7 @@
 
 import time
 
+import astor
 import six.moves.cPickle as pickle
 from six.moves import input
 from six.moves import xrange as range
@@ -21,6 +22,7 @@
 from model.reconstruction_model import Reconstructor
 from model.utils import GloveHelper
 
+assert astor.__version__ == "0.7.1"
 if six.PY3:
     # import additional packages for wikisql dataset (works only under Python 3)
     pass
@@ -146,7 +148,7 @@ def train(args):
                 print('[Epoch %d] begin validation' % epoch, file=sys.stderr)
                 eval_start = time.time()
                 eval_results = evaluation.evaluate(dev_set.examples, model, evaluator, args,
-                                                   verbose=True, eval_top_pred_only=args.eval_top_pred_only)
+                                                   verbose=False, eval_top_pred_only=args.eval_top_pred_only)
                 dev_score = eval_results[evaluator.default_metric]
 
                 print('[Epoch %d] evaluate details: %s, dev %s: %.5f (took %ds)' % (

model/nn_utils.py

@@ -3,9 +3,7 @@
 import torch
 import torch.nn.functional as F
 import torch.nn.init as init
-import numpy as np
 
-import torch
 import torch.nn as nn
 from torch.autograd import Variable
 import numpy as np

scripts/conala/finetune.sh

@@ -3,7 +3,9 @@ set -e
 
 seed=0
 mined_num=$1
-vocab="data/conala/vocab.src_freq3.code_freq3.mined_${mined_num}.bin"
+pretrained_model_name=$2
+freq=${3:-3}
+vocab="data/conala/vocab.src_freq${freq}.code_freq${freq}.mined_${mined_num}.bin"
 finetune_file="data/conala/train.bin"
 dev_file="data/conala/dev.bin"
 dropout=0.3
@@ -17,7 +19,6 @@ lr_decay=0.5
 beam_size=15
 lstm='lstm'  # lstm
 lr_decay_after_epoch=15
-pretrained_model_name=$2
 model_name=finetune.conala.${lstm}.hidden${hidden_size}.embed${embed_size}.action${action_embed_size}.field${field_embed_size}.type${type_embed_size}.dr${dropout}.lr${lr}.lr_de${lr_decay}.lr_da${lr_decay_after_epoch}.beam${beam_size}.seed${seed}.pre_${mined_num}
 
 echo "**** Writing results to logs/conala/${model_name}.log ****"

scripts/conala/vanilla.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-seed=0
+seed=${1:-0}
 vocab="data/conala/vocab.src_freq3.code_freq3.mined_0.bin"
 train_file="data/conala/train.mined_0.bin"
 dev_file="data/conala/dev.bin"