Fix ascii decode error

Author: Guanheng Zhang

beginner_source/torchtext_translation_tutorial.py

@@ -43,7 +43,7 @@
 from torchtext.data.utils import get_tokenizer
 from collections import Counter
 from torchtext.vocab import Vocab
-from torchtext.utils import download_from_url, extract_archive
+from torchtext.utils import download_from_url, extract_archive, unicode_csv_reader
 import io
 
 url_base = 'https://raw.githubusercontent.com/multi30k/dataset/master/data/task1/raw/'
@@ -69,13 +69,14 @@ def build_vocab(filepath, tokenizer):
 en_vocab = build_vocab(train_filepaths[1], en_tokenizer)
 
 def data_process(filepaths):
-  raw_de_iter, raw_en_iter = iter(io.open(filepaths[0])), iter(io.open(filepaths[1]))
+  raw_de_iter = iter(unicode_csv_reader(io.open(filepaths[0])))
+  raw_en_iter = iter(unicode_csv_reader(io.open(filepaths[1])))
   data = []
   for (raw_de, raw_en) in zip(raw_de_iter, raw_en_iter):
-    de_tensor_ = torch.tensor([de_vocab[token] for token in de_tokenizer(raw_de)],
-                            dtype=torch.long)
-    en_tensor_ = torch.tensor([en_vocab[token] for token in en_tokenizer(raw_en)],
-                            dtype=torch.long)
+    de_tensor_ = torch.tensor([de_vocab[token] for token in de_tokenizer(" ".join(raw_de))],
+                              dtype=torch.long)
+    en_tensor_ = torch.tensor([en_vocab[token] for token in en_tokenizer(" ".join(raw_en))],
+                              dtype=torch.long)
     data.append((de_tensor_, en_tensor_))
   return data