|
15 | 15 | package unicode |
16 | 16 |
|
17 | 17 | import ( |
| 18 | + "bytes" |
| 19 | + "io/ioutil" |
18 | 20 | "reflect" |
19 | 21 | "testing" |
20 | 22 |
|
@@ -151,52 +153,167 @@ func TestUnicode(t *testing.T) { |
151 | 153 | actual := tokenizer.Tokenize(test.input) |
152 | 154 |
|
153 | 155 | if !reflect.DeepEqual(actual, test.output) { |
154 | | - t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) |
| 156 | + t.Errorf("Expected\n%v\n, got\n%v\n for %q", test.output, actual, string(test.input)) |
155 | 157 | } |
156 | 158 | } |
157 | 159 | } |
158 | 160 |
|
159 | | -var sampleLargeInput = []byte(`There are three characteristics of liquids which are relevant to the discussion of a BLEVE: |
160 | | -If a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed). |
161 | | -The boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example. |
162 | | -When a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did. |
163 | | -Typically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, oxygen, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures. |
164 | | -If the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings.`) |
| 161 | +func TestPreviousIdentical(t *testing.T) { |
| 162 | + file, err := ioutil.ReadFile("testdata/sample.txt") |
| 163 | + if err != nil { |
| 164 | + t.Fatal(err) |
| 165 | + } |
| 166 | + |
| 167 | + previous := TokenizePrevious(file) |
| 168 | + current := NewUnicodeTokenizer().Tokenize(file) |
| 169 | + |
| 170 | + if len(previous) != len(current) { |
| 171 | + t.Errorf("len(previous): %d, len(current): %d\n", len(previous), len(current)) |
| 172 | + } |
| 173 | + |
| 174 | + for i := range previous { |
| 175 | + prev := previous[i] |
| 176 | + curr := current[i] |
| 177 | + if !bytes.Equal(prev.Term, curr.Term) { |
| 178 | + t.Fatalf("previous term: %q, current term: %q", prev.Term, curr.Term) |
| 179 | + } |
| 180 | + if prev.Start != curr.Start { |
| 181 | + t.Fatalf("prev.Start: %d, curr.Start: %d", prev.Start, curr.Start) |
| 182 | + } |
| 183 | + if prev.End != curr.End { |
| 184 | + t.Fatalf("prev.End: %d, curr.End: %d", prev.End, curr.End) |
| 185 | + } |
| 186 | + if prev.Position != curr.Position { |
| 187 | + t.Fatalf("prev.Position: %d, curr.Position: %d", prev.Position, curr.Position) |
| 188 | + } |
| 189 | + if prev.Type != curr.Type { |
| 190 | + t.Errorf("prev.Type: %v, curr.Type: %v\n\n", prev.Type, curr.Type) |
| 191 | + } |
| 192 | + } |
| 193 | +} |
165 | 194 |
|
166 | | -func BenchmarkTokenizeEnglishText(b *testing.B) { |
| 195 | +func BenchmarkTokenizeMultilingual(b *testing.B) { |
| 196 | + file, err := ioutil.ReadFile("testdata/sample.txt") |
| 197 | + if err != nil { |
| 198 | + b.Fatal(err) |
| 199 | + } |
167 | 200 |
|
168 | 201 | tokenizer := NewUnicodeTokenizer() |
169 | | - b.ResetTimer() |
170 | 202 |
|
| 203 | + b.SetBytes(int64(len(file))) |
| 204 | + |
| 205 | + b.ResetTimer() |
171 | 206 | for i := 0; i < b.N; i++ { |
172 | | - tokenizer.Tokenize(sampleLargeInput) |
| 207 | + tokens := tokenizer.Tokenize(file) |
| 208 | + b.ReportMetric(float64(len(tokens)), "tokens") |
173 | 209 | } |
| 210 | +} |
174 | 211 |
|
| 212 | +func BenchmarkTokenizeMultilingualPrevious(b *testing.B) { |
| 213 | + file, err := ioutil.ReadFile("testdata/sample.txt") |
| 214 | + if err != nil { |
| 215 | + b.Fatal(err) |
| 216 | + } |
| 217 | + |
| 218 | + b.SetBytes(int64(len(file))) |
| 219 | + |
| 220 | + b.ResetTimer() |
| 221 | + for i := 0; i < b.N; i++ { |
| 222 | + tokens := TokenizePrevious(file) |
| 223 | + b.ReportMetric(float64(len(tokens)), "tokens") |
| 224 | + } |
175 | 225 | } |
176 | 226 |
|
177 | | -func TestConvertType(t *testing.T) { |
178 | | - tests := []struct { |
179 | | - in int |
180 | | - out analysis.TokenType |
181 | | - }{ |
182 | | - { |
183 | | - segment.Ideo, analysis.Ideographic, |
184 | | - }, |
185 | | - { |
186 | | - segment.Kana, analysis.Ideographic, |
187 | | - }, |
188 | | - { |
189 | | - segment.Number, analysis.Numeric, |
190 | | - }, |
191 | | - { |
192 | | - segment.Letter, analysis.AlphaNumeric, |
193 | | - }, |
| 227 | +// Previous implementation for testing ↓ |
| 228 | + |
| 229 | +func TokenizePrevious(input []byte) analysis.TokenStream { |
| 230 | + rvx := make([]analysis.TokenStream, 0, 10) // When rv gets full, append to rvx. |
| 231 | + rv := make(analysis.TokenStream, 0, 1) |
| 232 | + |
| 233 | + ta := []analysis.Token(nil) |
| 234 | + taNext := 0 |
| 235 | + |
| 236 | + segmenter := segment.NewWordSegmenterDirect(input) |
| 237 | + start := 0 |
| 238 | + pos := 1 |
| 239 | + |
| 240 | + guessRemaining := func(end int) int { |
| 241 | + avgSegmentLen := end / (len(rv) + 1) |
| 242 | + if avgSegmentLen < 1 { |
| 243 | + avgSegmentLen = 1 |
| 244 | + } |
| 245 | + |
| 246 | + remainingLen := len(input) - end |
| 247 | + |
| 248 | + return remainingLen / avgSegmentLen |
194 | 249 | } |
195 | 250 |
|
196 | | - for _, test := range tests { |
197 | | - actual := convertType(test.in) |
198 | | - if actual != test.out { |
199 | | - t.Errorf("expected %d, got %d for %d", test.out, actual, test.in) |
| 251 | + for segmenter.Segment() { |
| 252 | + segmentBytes := segmenter.Bytes() |
| 253 | + end := start + len(segmentBytes) |
| 254 | + if segmenter.Type() != segment.None { |
| 255 | + if taNext >= len(ta) { |
| 256 | + remainingSegments := guessRemaining(end) |
| 257 | + if remainingSegments > 1000 { |
| 258 | + remainingSegments = 1000 |
| 259 | + } |
| 260 | + if remainingSegments < 1 { |
| 261 | + remainingSegments = 1 |
| 262 | + } |
| 263 | + |
| 264 | + ta = make([]analysis.Token, remainingSegments) |
| 265 | + taNext = 0 |
| 266 | + } |
| 267 | + |
| 268 | + token := &ta[taNext] |
| 269 | + taNext++ |
| 270 | + |
| 271 | + token.Term = segmentBytes |
| 272 | + token.Start = start |
| 273 | + token.End = end |
| 274 | + token.Position = pos |
| 275 | + token.Type = convertType(segmenter.Type()) |
| 276 | + |
| 277 | + if len(rv) >= cap(rv) { // When rv is full, save it into rvx. |
| 278 | + rvx = append(rvx, rv) |
| 279 | + |
| 280 | + rvCap := cap(rv) * 2 |
| 281 | + if rvCap > 256 { |
| 282 | + rvCap = 256 |
| 283 | + } |
| 284 | + |
| 285 | + rv = make(analysis.TokenStream, 0, rvCap) // Next rv cap is bigger. |
| 286 | + } |
| 287 | + |
| 288 | + rv = append(rv, token) |
| 289 | + pos++ |
| 290 | + } |
| 291 | + start = end |
| 292 | + } |
| 293 | + |
| 294 | + if len(rvx) > 0 { |
| 295 | + n := len(rv) |
| 296 | + for _, r := range rvx { |
| 297 | + n += len(r) |
200 | 298 | } |
| 299 | + rall := make(analysis.TokenStream, 0, n) |
| 300 | + for _, r := range rvx { |
| 301 | + rall = append(rall, r...) |
| 302 | + } |
| 303 | + return append(rall, rv...) |
| 304 | + } |
| 305 | + |
| 306 | + return rv |
| 307 | +} |
| 308 | + |
| 309 | +func convertType(segmentWordType int) analysis.TokenType { |
| 310 | + switch segmentWordType { |
| 311 | + case segment.Ideo: |
| 312 | + return analysis.Ideographic |
| 313 | + case segment.Kana: |
| 314 | + return analysis.Ideographic |
| 315 | + case segment.Number: |
| 316 | + return analysis.Numeric |
201 | 317 | } |
| 318 | + return analysis.AlphaNumeric |
202 | 319 | } |
0 commit comments