-
Notifications
You must be signed in to change notification settings - Fork 0
/
verificationMethods.py
126 lines (101 loc) · 5.58 KB
/
verificationMethods.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
import nltk
import openpyxl
import random
import warnings
from commonTranslator import CommonTranslator
from cosine import calculateCosineSimilarity
from fileManager import FileManager
from levDist import levenshteinDistance
from nltk.translate.bleu_score import SmoothingFunction
"""
TestCaseManager
"""
class TestCaseManager:
def __init__(self, results_file = 'testResults.xlsx'):
self._translator = CommonTranslator()
self._translation_file = FileManager()
self._results_file = FileManager(results_file,
[['Service', 'Test Phrase', 'Source Language', 'Intermediate Language', 'Target Language', 'Direct Translation', 'Alternate Translation'],
['Service', 'Source Language', 'Target Language', 'Lev-Distance', 'BLEU-Score', 'Cosine-Similarity']],
['Test Phrases', 'Test Results'],
write_header = True)
# Assumes structure of results file
self._language_list = self._translation_file.getColumnNames('Google')
"""
Verifies a particular translation by performing one additional 'side-translation'
and comparing the result.
The results of each test are then stored in a file.
TODO: Handle incomplete data file.
"""
def uniDirectionalTest(self, services, target_language, rows = None, origin_language = 'en'):
if rows is None:
rows = range(1, self._translation_file.getNumberRows('Google'))
for service in services:
print('Performing', service, 'translations..')
for row in rows:
print(' Analysing row', row)
remaining_languages = [x for x in self._language_list if x != origin_language and x != target_language]
# Randomly select a side-language and grab its direct translation.
random_language = random.choice(remaining_languages)
# Grab the original, direct, and side-translations from file.
original_phrase, direct_translation, intermediate_translation = self._translation_file.readRow(service, row, [origin_language, target_language, random_language])
# Translate the intermediate sentence into the same language as the direct translation.
alternate_translation = self._translator.translate(service, intermediate_translation, target_language, random_language)
# If failed translation, keep trying with different selection
while len(remaining_languages) > 1 and alternate_translation is None:
print('WARNING: Failed to translate from', random_language, 'to', target_language, '. Repeated failures may introduce biased results.')
remaining_languages.remove(random_language)
random_language = random.choice(remaining_languages)
alternate_translation = self._translator.translate(service, intermediate_translation, target_language, random_language)
# Check for complete failure.
if alternate_translation is None:
print('Unable to perform side-translation. Recording as N/A..')
self._results_file.appendEntry('Test Phrases', [service, original_phrase, origin_language, 'N/A', target_language, direct_translation, 'N/A'])
self._results_file.appendEntry('Test Results', [service, origin_language, target_language, 'N/A', 'N/A', 'N/A'])
else:
# Compare the two translations.
score = self.compare(direct_translation, alternate_translation)
# Record the results.
self._results_file.appendEntry('Test Phrases', [service, original_phrase, origin_language, random_language, target_language, direct_translation, alternate_translation])
self._results_file.appendEntry('Test Results', [service, origin_language, target_language, score['Lev-Distance'], score['BLEU-Score'], score['Cosine-Similarity']])
print('Saving Progress..')
self._results_file.save()
"""
Verifies a particular translation by performing a reverse translation back
to the origin lannguage.
NOTE: This is simply a special case of the uniDirectionalTest where the
target language is also the origin language.
"""
def biDirectionalTest(self, services, rows, origin_language = 'en'):
self.uniDirectionalTest(services, rows, origin_language, origin_language)
"""
Compares two strings and returns a score based on their similarity.
Can optionally specify which metrics to use
TODO: Incorporate more than one test result.
"""
def compare(self, phrase1, phrase2, metrics = None):
if metrics is None:
metrics = ['Lev-Distance', 'BLEU-Score', 'Cosine-Similarity']
results = dict()
if 'Lev-Distance' in metrics:
results['Lev-Distance'] = levenshteinDistance(phrase1, phrase2)['Ratio']
if 'BLEU-Score' in metrics:
results['BLEU-Score'] = nltk.translate.bleu_score.sentence_bleu([phrase1.split()], phrase2.split(), smoothing_function = SmoothingFunction().method2)
if 'Cosine-Similarity' in metrics:
results['Cosine-Similarity'] = calculateCosineSimilarity(phrase1, phrase2)
return results
"""
Performs a single metric calculation for all rows in an existing test case file.
Primarily used for re-calculating scores without having to run any additional translations.
"""
def recomputeMetricResults(self, metric, rows = None):
if rows is None:
rows = range(2, self._results_file.getNumberRows('Test Phrases') + 1)
for row in rows:
# Get the sample strings for this observation.
direct_translation, alternate_translation = self._results_file.readRow('Test Phrases', row, ['Direct Translation', 'Alternate Translation'])
# Compute a metric for observation
score = self.compare(direct_translation, alternate_translation, [metric])
# Write result to file
self._results_file.writeRow('Test Results', row, [score[metric]], [metric])
self._results_file.save()