Skip to content

Commit 9ae0815

Browse files
CyrusNajmabadiCyrusNajmabadi
committed
Treat 0x0085 as whitespace, not as a line terminator. This matches ES5 and ES6.
1 parent 0d2a5bb commit 9ae0815

13 files changed

+167
-87
lines changed

‎src/compiler/scanner.ts‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -318,13 +318,38 @@ module ts {
318318
let hasOwnProperty = Object.prototype.hasOwnProperty;
319319

320320
export function isWhiteSpace(ch: number): boolean {
321-
return ch === CharacterCodes.space || ch === CharacterCodes.tab || ch === CharacterCodes.verticalTab || ch === CharacterCodes.formFeed ||
322-
ch === CharacterCodes.nonBreakingSpace || ch === CharacterCodes.ogham || ch >= CharacterCodes.enQuad && ch <= CharacterCodes.zeroWidthSpace ||
323-
ch === CharacterCodes.narrowNoBreakSpace || ch === CharacterCodes.mathematicalSpace || ch === CharacterCodes.ideographicSpace || ch === CharacterCodes.byteOrderMark;
321+
// Note: nextLine is in the Zs space, and should be considered to be a whitespace.
322+
// It is explicitly not a line-break as it isn't in the exact set specified by EcmaScript.
323+
return ch === CharacterCodes.space ||
324+
ch === CharacterCodes.tab ||
325+
ch === CharacterCodes.verticalTab ||
326+
ch === CharacterCodes.formFeed ||
327+
ch === CharacterCodes.nonBreakingSpace ||
328+
ch === CharacterCodes.nextLine ||
329+
ch === CharacterCodes.ogham ||
330+
ch >= CharacterCodes.enQuad && ch <= CharacterCodes.zeroWidthSpace ||
331+
ch === CharacterCodes.narrowNoBreakSpace ||
332+
ch === CharacterCodes.mathematicalSpace ||
333+
ch === CharacterCodes.ideographicSpace ||
334+
ch === CharacterCodes.byteOrderMark;
324335
}
325336

326337
export function isLineBreak(ch: number): boolean {
327-
return ch === CharacterCodes.lineFeed || ch === CharacterCodes.carriageReturn || ch === CharacterCodes.lineSeparator || ch === CharacterCodes.paragraphSeparator || ch === CharacterCodes.nextLine;
338+
// ES5 7.3:
339+
// The ECMAScript line terminator characters are listed in Table 3.
340+
// Table 3 — Line Terminator Characters
341+
// Code Unit Value Name Formal Name
342+
// \u000A Line Feed <LF>
343+
// \u000D Carriage Return <CR>
344+
// \u2028 Line separator <LS>
345+
// \u2029 Paragraph separator <PS>
346+
// Only the characters in Table 3 are treated as line terminators. Other new line or line
347+
// breaking characters are treated as white space but not as line terminators.
348+
349+
return ch === CharacterCodes.lineFeed ||
350+
ch === CharacterCodes.carriageReturn ||
351+
ch === CharacterCodes.lineSeparator ||
352+
ch === CharacterCodes.paragraphSeparator;
328353
}
329354

330355
function isDigit(ch: number): boolean {

‎tests/baselines/reference/fileWithNextLine1.js‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
//// [fileWithNextLine1.ts]
2+
// Note: there is a nextline (0x85) in the string
3+
// 0. It should be counted as a space and should not cause an error.
4+
var v = '…';
5+
6+
//// [fileWithNextLine1.js]
7+
// Note: there is a nextline (0x85) in the string
8+
// 0. It should be counted as a space and should not cause an error.
9+
var v = '…';

‎tests/baselines/reference/fileWithNextLine1.types‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
=== tests/cases/compiler/fileWithNextLine1.ts ===
2+
// Note: there is a nextline (0x85) in the string
3+
// 0. It should be counted as a space and should not cause an error.
4+
var v = '…';
5+
>v : string
6+

‎tests/baselines/reference/fileWithNextLine2.js‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
//// [fileWithNextLine2.ts]
2+
// Note: there is a nextline (0x85) char between the = and the 0.
3+
// it should be treated like a space
4+
var v =…0;
5+
6+
//// [fileWithNextLine2.js]
7+
// Note: there is a nextline (0x85) char between the = and the 0.
8+
// it should be treated like a space
9+
var v = 0;

‎tests/baselines/reference/fileWithNextLine2.types‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
=== tests/cases/compiler/fileWithNextLine2.ts ===
2+
// Note: there is a nextline (0x85) char between the = and the 0.
3+
// it should be treated like a space
4+
var v =…0;
5+
>v : number
6+

‎tests/baselines/reference/fileWithNextLine3.errors.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
tests/cases/compiler/fileWithNextLine3.ts(3,1): error TS1108: A 'return' statement can only be used within a function body.
2+
3+
4+
==== tests/cases/compiler/fileWithNextLine3.ts (1 errors) ====
5+
// Note: there is a nextline (0x85) between the return and the
6+
// 0. It should be counted as a space and should not trigger ASI
7+
return…0;
8+
~~~~~~
9+
!!! error TS1108: A 'return' statement can only be used within a function body.

‎tests/baselines/reference/fileWithNextLine3.js‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
//// [fileWithNextLine3.ts]
2+
// Note: there is a nextline (0x85) between the return and the
3+
// 0. It should be counted as a space and should not trigger ASI
4+
return…0;
5+
6+
//// [fileWithNextLine3.js]
7+
// Note: there is a nextline (0x85) between the return and the
8+
// 0. It should be counted as a space and should not trigger ASI
9+
return 0;

‎tests/baselines/reference/sourceMap-LineBreaks.js.map‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/baselines/reference/sourceMap-LineBreaks.sourcemap.txt‎

Lines changed: 74 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -78,18 +78,18 @@ sourceFile:sourceMap-LineBreaks.ts
7878
5 > ^
7979
6 > ^
8080
7 > ^^^^^^^^^^^^^^^->
81-
1->… >
81+
1->…
8282
2 >var
8383
3 > endsWithLineFeed
8484
4 > =
8585
5 > 1
8686
6 > ;
87-
1->Emitted(4, 1) Source(4, 1) + SourceIndex(0)
88-
2 >Emitted(4, 5) Source(4, 5) + SourceIndex(0)
89-
3 >Emitted(4, 21) Source(4, 21) + SourceIndex(0)
90-
4 >Emitted(4, 24) Source(4, 24) + SourceIndex(0)
91-
5 >Emitted(4, 25) Source(4, 25) + SourceIndex(0)
92-
6 >Emitted(4, 26) Source(4, 26) + SourceIndex(0)
87+
1->Emitted(4, 1) Source(3, 27) + SourceIndex(0)
88+
2 >Emitted(4, 5) Source(3, 31) + SourceIndex(0)
89+
3 >Emitted(4, 21) Source(3, 47) + SourceIndex(0)
90+
4 >Emitted(4, 24) Source(3, 50) + SourceIndex(0)
91+
5 >Emitted(4, 25) Source(3, 51) + SourceIndex(0)
92+
6 >Emitted(4, 26) Source(3, 52) + SourceIndex(0)
9393
---
9494
>>>var endsWithCarriageReturnLineFeed = 1;
9595
1->
@@ -105,12 +105,12 @@ sourceFile:sourceMap-LineBreaks.ts
105105
4 > =
106106
5 > 1
107107
6 > ;
108-
1->Emitted(5, 1) Source(5, 1) + SourceIndex(0)
109-
2 >Emitted(5, 5) Source(5, 5) + SourceIndex(0)
110-
3 >Emitted(5, 35) Source(5, 35) + SourceIndex(0)
111-
4 >Emitted(5, 38) Source(5, 38) + SourceIndex(0)
112-
5 >Emitted(5, 39) Source(5, 39) + SourceIndex(0)
113-
6 >Emitted(5, 40) Source(5, 40) + SourceIndex(0)
108+
1->Emitted(5, 1) Source(4, 1) + SourceIndex(0)
109+
2 >Emitted(5, 5) Source(4, 5) + SourceIndex(0)
110+
3 >Emitted(5, 35) Source(4, 35) + SourceIndex(0)
111+
4 >Emitted(5, 38) Source(4, 38) + SourceIndex(0)
112+
5 >Emitted(5, 39) Source(4, 39) + SourceIndex(0)
113+
6 >Emitted(5, 40) Source(4, 40) + SourceIndex(0)
114114
---
115115
>>>var endsWithCarriageReturn = 1;
116116
1 >
@@ -127,12 +127,12 @@ sourceFile:sourceMap-LineBreaks.ts
127127
4 > =
128128
5 > 1
129129
6 > ;
130-
1 >Emitted(6, 1) Source(6, 1) + SourceIndex(0)
131-
2 >Emitted(6, 5) Source(6, 5) + SourceIndex(0)
132-
3 >Emitted(6, 27) Source(6, 27) + SourceIndex(0)
133-
4 >Emitted(6, 30) Source(6, 30) + SourceIndex(0)
134-
5 >Emitted(6, 31) Source(6, 31) + SourceIndex(0)
135-
6 >Emitted(6, 32) Source(6, 32) + SourceIndex(0)
130+
1 >Emitted(6, 1) Source(5, 1) + SourceIndex(0)
131+
2 >Emitted(6, 5) Source(5, 5) + SourceIndex(0)
132+
3 >Emitted(6, 27) Source(5, 27) + SourceIndex(0)
133+
4 >Emitted(6, 30) Source(5, 30) + SourceIndex(0)
134+
5 >Emitted(6, 31) Source(5, 31) + SourceIndex(0)
135+
6 >Emitted(6, 32) Source(5, 32) + SourceIndex(0)
136136
---
137137
>>>var endsWithLineFeedCarriageReturn = 1;
138138
1->
@@ -148,12 +148,12 @@ sourceFile:sourceMap-LineBreaks.ts
148148
4 > =
149149
5 > 1
150150
6 > ;
151-
1->Emitted(7, 1) Source(7, 1) + SourceIndex(0)
152-
2 >Emitted(7, 5) Source(7, 5) + SourceIndex(0)
153-
3 >Emitted(7, 35) Source(7, 35) + SourceIndex(0)
154-
4 >Emitted(7, 38) Source(7, 38) + SourceIndex(0)
155-
5 >Emitted(7, 39) Source(7, 39) + SourceIndex(0)
156-
6 >Emitted(7, 40) Source(7, 40) + SourceIndex(0)
151+
1->Emitted(7, 1) Source(6, 1) + SourceIndex(0)
152+
2 >Emitted(7, 5) Source(6, 5) + SourceIndex(0)
153+
3 >Emitted(7, 35) Source(6, 35) + SourceIndex(0)
154+
4 >Emitted(7, 38) Source(6, 38) + SourceIndex(0)
155+
5 >Emitted(7, 39) Source(6, 39) + SourceIndex(0)
156+
6 >Emitted(7, 40) Source(6, 40) + SourceIndex(0)
157157
---
158158
>>>var endsWithLineFeedCarriageReturnLineFeed = 1;
159159
1->
@@ -169,12 +169,12 @@ sourceFile:sourceMap-LineBreaks.ts
169169
4 > =
170170
5 > 1
171171
6 > ;
172-
1->Emitted(8, 1) Source(9, 1) + SourceIndex(0)
173-
2 >Emitted(8, 5) Source(9, 5) + SourceIndex(0)
174-
3 >Emitted(8, 43) Source(9, 43) + SourceIndex(0)
175-
4 >Emitted(8, 46) Source(9, 46) + SourceIndex(0)
176-
5 >Emitted(8, 47) Source(9, 47) + SourceIndex(0)
177-
6 >Emitted(8, 48) Source(9, 48) + SourceIndex(0)
172+
1->Emitted(8, 1) Source(8, 1) + SourceIndex(0)
173+
2 >Emitted(8, 5) Source(8, 5) + SourceIndex(0)
174+
3 >Emitted(8, 43) Source(8, 43) + SourceIndex(0)
175+
4 >Emitted(8, 46) Source(8, 46) + SourceIndex(0)
176+
5 >Emitted(8, 47) Source(8, 47) + SourceIndex(0)
177+
6 >Emitted(8, 48) Source(8, 48) + SourceIndex(0)
178178
---
179179
>>>var stringLiteralWithLineFeed = "line 1\
180180
1 >
@@ -187,10 +187,10 @@ sourceFile:sourceMap-LineBreaks.ts
187187
2 >var
188188
3 > stringLiteralWithLineFeed
189189
4 > =
190-
1 >Emitted(9, 1) Source(11, 1) + SourceIndex(0)
191-
2 >Emitted(9, 5) Source(11, 5) + SourceIndex(0)
192-
3 >Emitted(9, 30) Source(11, 30) + SourceIndex(0)
193-
4 >Emitted(9, 33) Source(11, 33) + SourceIndex(0)
190+
1 >Emitted(9, 1) Source(10, 1) + SourceIndex(0)
191+
2 >Emitted(9, 5) Source(10, 5) + SourceIndex(0)
192+
3 >Emitted(9, 30) Source(10, 30) + SourceIndex(0)
193+
4 >Emitted(9, 33) Source(10, 33) + SourceIndex(0)
194194
---
195195
>>>line 2";
196196
1 >^^^^^^^
@@ -199,8 +199,8 @@ sourceFile:sourceMap-LineBreaks.ts
199199
1 >"line 1\
200200
>line 2"
201201
2 > ;
202-
1 >Emitted(10, 8) Source(12, 8) + SourceIndex(0)
203-
2 >Emitted(10, 9) Source(12, 9) + SourceIndex(0)
202+
1 >Emitted(10, 8) Source(11, 8) + SourceIndex(0)
203+
2 >Emitted(10, 9) Source(11, 9) + SourceIndex(0)
204204
---
205205
>>>var stringLiteralWithCarriageReturnLineFeed = "line 1\
206206
1->
@@ -212,10 +212,10 @@ sourceFile:sourceMap-LineBreaks.ts
212212
2 >var
213213
3 > stringLiteralWithCarriageReturnLineFeed
214214
4 > =
215-
1->Emitted(11, 1) Source(13, 1) + SourceIndex(0)
216-
2 >Emitted(11, 5) Source(13, 5) + SourceIndex(0)
217-
3 >Emitted(11, 44) Source(13, 44) + SourceIndex(0)
218-
4 >Emitted(11, 47) Source(13, 47) + SourceIndex(0)
215+
1->Emitted(11, 1) Source(12, 1) + SourceIndex(0)
216+
2 >Emitted(11, 5) Source(12, 5) + SourceIndex(0)
217+
3 >Emitted(11, 44) Source(12, 44) + SourceIndex(0)
218+
4 >Emitted(11, 47) Source(12, 47) + SourceIndex(0)
219219
---
220220
>>>line 2";
221221
1 >^^^^^^^
@@ -224,8 +224,8 @@ sourceFile:sourceMap-LineBreaks.ts
224224
1 >"line 1\
225225
>line 2"
226226
2 > ;
227-
1 >Emitted(12, 8) Source(14, 8) + SourceIndex(0)
228-
2 >Emitted(12, 9) Source(14, 9) + SourceIndex(0)
227+
1 >Emitted(12, 8) Source(13, 8) + SourceIndex(0)
228+
2 >Emitted(12, 9) Source(13, 9) + SourceIndex(0)
229229
---
230230
>>>var stringLiteralWithCarriageReturn = "line 1\1->
231231
2 >^^^^
@@ -236,19 +236,19 @@ sourceFile:sourceMap-LineBreaks.ts
236236
2 >var
237237
3 > stringLiteralWithCarriageReturn
238238
4 > =
239-
1->Emitted(13, 1) Source(15, 1) + SourceIndex(0)
240-
2 >Emitted(13, 5) Source(15, 5) + SourceIndex(0)
241-
3 >Emitted(13, 36) Source(15, 36) + SourceIndex(0)
242-
4 >Emitted(13, 39) Source(15, 39) + SourceIndex(0)
239+
1->Emitted(13, 1) Source(14, 1) + SourceIndex(0)
240+
2 >Emitted(13, 5) Source(14, 5) + SourceIndex(0)
241+
3 >Emitted(13, 36) Source(14, 36) + SourceIndex(0)
242+
4 >Emitted(13, 39) Source(14, 39) + SourceIndex(0)
243243
---
244244
>>>line 2";
245245
1 >^^^^^^^
246246
2 > ^
247247
3 > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^->
248248
1 >"line 1\ >line 2"
249249
2 > ;
250-
1 >Emitted(14, 8) Source(16, 8) + SourceIndex(0)
251-
2 >Emitted(14, 9) Source(16, 9) + SourceIndex(0)
250+
1 >Emitted(14, 8) Source(15, 8) + SourceIndex(0)
251+
2 >Emitted(14, 9) Source(15, 9) + SourceIndex(0)
252252
---
253253
>>>var stringLiteralWithLineSeparator = "line 1\
1->
254254
2 >^^^^
@@ -260,19 +260,19 @@ sourceFile:sourceMap-LineBreaks.ts
260260
2 >var
261261
3 > stringLiteralWithLineSeparator
262262
4 > =
263-
1->Emitted(15, 1) Source(18, 1) + SourceIndex(0)
264-
2 >Emitted(15, 5) Source(18, 5) + SourceIndex(0)
265-
3 >Emitted(15, 35) Source(18, 35) + SourceIndex(0)
266-
4 >Emitted(15, 38) Source(18, 38) + SourceIndex(0)
263+
1->Emitted(15, 1) Source(17, 1) + SourceIndex(0)
264+
2 >Emitted(15, 5) Source(17, 5) + SourceIndex(0)
265+
3 >Emitted(15, 35) Source(17, 35) + SourceIndex(0)
266+
4 >Emitted(15, 38) Source(17, 38) + SourceIndex(0)
267267
---
268268
>>>line 2";
269269
1 >^^^^^^^
270270
2 > ^
271271
3 > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^->
272272
1 >"line 1\
 >line 2"
273273
2 > ;
274-
1 >Emitted(16, 8) Source(19, 8) + SourceIndex(0)
275-
2 >Emitted(16, 9) Source(19, 9) + SourceIndex(0)
274+
1 >Emitted(16, 8) Source(18, 8) + SourceIndex(0)
275+
2 >Emitted(16, 9) Source(18, 9) + SourceIndex(0)
276276
---
277277
>>>var stringLiteralWithParagraphSeparator = "line 1\
1->
278278
2 >^^^^
@@ -282,40 +282,38 @@ sourceFile:sourceMap-LineBreaks.ts
282282
2 >var
283283
3 > stringLiteralWithParagraphSeparator
284284
4 > =
285-
1->Emitted(17, 1) Source(20, 1) + SourceIndex(0)
286-
2 >Emitted(17, 5) Source(20, 5) + SourceIndex(0)
287-
3 >Emitted(17, 40) Source(20, 40) + SourceIndex(0)
288-
4 >Emitted(17, 43) Source(20, 43) + SourceIndex(0)
285+
1->Emitted(17, 1) Source(19, 1) + SourceIndex(0)
286+
2 >Emitted(17, 5) Source(19, 5) + SourceIndex(0)
287+
3 >Emitted(17, 40) Source(19, 40) + SourceIndex(0)
288+
4 >Emitted(17, 43) Source(19, 43) + SourceIndex(0)
289289
---
290290
>>>line 2";
291291
1 >^^^^^^^
292292
2 > ^
293-
3 > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^->
293+
3 > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^->
294294
1 >"line 1\
 >line 2"
295295
2 > ;
296-
1 >Emitted(18, 8) Source(21, 8) + SourceIndex(0)
297-
2 >Emitted(18, 9) Source(21, 9) + SourceIndex(0)
296+
1 >Emitted(18, 8) Source(20, 8) + SourceIndex(0)
297+
2 >Emitted(18, 9) Source(20, 9) + SourceIndex(0)
298298
---
299-
>>>var stringLiteralWithNextLine = "line 1\…1->
299+
>>>var stringLiteralWithNextLine = "line 1\…line 2";
300+
1->
300301
2 >^^^^
301302
3 > ^^^^^^^^^^^^^^^^^^^^^^^^^
302303
4 > ^^^
304+
5 > ^^^^^^^^^^^^^^^^
305+
6 > ^
303306
1->
 >
304307
2 >var
305308
3 > stringLiteralWithNextLine
306309
4 > =
307-
1->Emitted(19, 1) Source(22, 1) + SourceIndex(0)
308-
2 >Emitted(19, 5) Source(22, 5) + SourceIndex(0)
309-
3 >Emitted(19, 30) Source(22, 30) + SourceIndex(0)
310-
4 >Emitted(19, 33) Source(22, 33) + SourceIndex(0)
311-
---
312-
>>>line 2";
313-
1 >^^^^^^^
314-
2 > ^
315-
3 > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^->
316-
1 >"line 1\… >line 2"
317-
2 > ;
318-
1 >Emitted(20, 8) Source(23, 8) + SourceIndex(0)
319-
2 >Emitted(20, 9) Source(23, 9) + SourceIndex(0)
310+
5 > "line 1\…line 2"
311+
6 > ;
312+
1->Emitted(19, 1) Source(21, 1) + SourceIndex(0)
313+
2 >Emitted(19, 5) Source(21, 5) + SourceIndex(0)
314+
3 >Emitted(19, 30) Source(21, 30) + SourceIndex(0)
315+
4 >Emitted(19, 33) Source(21, 33) + SourceIndex(0)
316+
5 >Emitted(19, 49) Source(21, 49) + SourceIndex(0)
317+
6 >Emitted(19, 50) Source(21, 50) + SourceIndex(0)
320318
---
321319
>>>//# sourceMappingURL=sourceMap-LineBreaks.js.map

0 commit comments

Comments
 (0)