dlangui.dml.dmlhighlight source code

1 module dlangui.dml.dmlhighlight;
2 
3 import dlangui.core.editable;
4 import dlangui.core.linestream;
5 import dlangui.core.textsource;
6 import dlangui.core.logger;
7 import dlangui.dml.parser;
8 import dlangui.widgets.metadata;
9 
10 class DMLSyntaxSupport : SyntaxSupport {
11 
12     EditableContent _content;
13     SourceFile _file;
14     this (string filename) {
15         _file = new SourceFile(filename);
16     }
17 
18     TokenPropString[] _props;
19 
20     /// returns editable content
21     @property EditableContent content() { return _content; }
22     /// set editable content
23     @property SyntaxSupport content(EditableContent content) {
24         _content = content;
25         return this;
26     }
27 
28     private enum BracketMatch {
29         CONTINUE,
30         FOUND,
31         ERROR
32     }
33 
34     private static struct BracketStack {
35         dchar[] buf;
36         int pos;
37         bool reverse;
38         void initialize(bool reverse) {
39             this.reverse = reverse;
40             pos = 0;
41         }
42         void push(dchar ch) {
43             if (buf.length <= pos)
44                 buf.length = pos + 16;
45             buf[pos++] = ch;
46         }
47         dchar pop() {
48             if (pos <= 0)
49                 return 0;
50             return buf[--pos];
51         }
52         BracketMatch process(dchar ch) {
53             if (reverse) {
54                 if (isCloseBracket(ch)) {
55                     push(ch);
56                     return BracketMatch.CONTINUE;
57                 } else {
58                     if (pop() != pairedBracket(ch))
59                         return BracketMatch.ERROR;
60                     if (pos == 0)
61                         return BracketMatch.FOUND;
62                     return BracketMatch.CONTINUE;
63                 }
64             } else {
65                 if (isOpenBracket(ch)) {
66                     push(ch);
67                     return BracketMatch.CONTINUE;
68                 } else {
69                     if (pop() != pairedBracket(ch))
70                         return BracketMatch.ERROR;
71                     if (pos == 0)
72                         return BracketMatch.FOUND;
73                     return BracketMatch.CONTINUE;
74                 }
75             }
76         }
77     }
78     BracketStack _bracketStack;
79     static bool isBracket(dchar ch) {
80         return pairedBracket(ch) != 0;
81     }
82     static dchar pairedBracket(dchar ch) {
83         switch (ch) {
84             case '(':
85                 return ')';
86             case ')':
87                 return '(';
88             case '{':
89                 return '}';
90             case '}':
91                 return '{';
92             case '[':
93                 return ']';
94             case ']':
95                 return '[';
96             default:
97                 return 0; // not a bracket
98         }
99     }
100     static bool isOpenBracket(dchar ch) {
101         switch (ch) {
102             case '(':
103             case '{':
104             case '[':
105                 return true;
106             default:
107                 return false;
108         }
109     }
110     static bool isCloseBracket(dchar ch) {
111         switch (ch) {
112             case ')':
113             case '}':
114             case ']':
115                 return true;
116             default:
117                 return false;
118         }
119     }
120 
121     protected dchar nextBracket(int dir, ref TextPosition p) {
122         for (;;) {
123             TextPosition oldpos = p;
124             p = dir < 0 ? _content.prevCharPos(p) : _content.nextCharPos(p);
125             if (p == oldpos)
126                 return 0;
127             auto prop = _content.tokenProp(p);
128             if (tokenCategory(prop) == TokenCategory.Op) {
129                 dchar ch = _content[p];
130                 if (isBracket(ch))
131                     return ch;
132             }
133         }
134     }
135 
136     /// returns paired bracket {} () [] for char at position p, returns paired char position or p if not found or not bracket
137     override TextPosition findPairedBracket(TextPosition p) {
138         if (p.line < 0 || p.line >= content.length)
139             return p;
140         dstring s = content.line(p.line);
141         if (p.pos < 0 || p.pos >= s.length)
142             return p;
143         dchar ch = content[p];
144         dchar paired = pairedBracket(ch);
145         if (!paired)
146             return p;
147         TextPosition startPos = p;
148         int dir = isOpenBracket(ch) ? 1 : -1;
149         _bracketStack.initialize(dir < 0);
150         _bracketStack.process(ch);
151         for (;;) {
152             ch = nextBracket(dir, p);
153             if (!ch) // no more brackets
154                 return startPos;
155             auto match = _bracketStack.process(ch);
156             if (match == BracketMatch.FOUND)
157                 return p;
158             if (match == BracketMatch.ERROR)
159                 return startPos;
160             // continue
161         }
162     }
163 
164 
165     /// return true if toggle line comment is supported for file type
166     override @property bool supportsToggleLineComment() {
167         return true;
168     }
169 
170     /// return true if can toggle line comments for specified text range
171     override bool canToggleLineComment(TextRange range) {
172         TextRange r = content.fullLinesRange(range);
173         if (isInsideBlockComment(r.start) || isInsideBlockComment(r.end))
174             return false;
175         return true;
176     }
177 
178     protected bool isLineComment(dstring s) {
179         foreach(i; 0 .. s.length - 1) {
180             if (s[i] == '/' && s[i + 1] == '/')
181                 return true;
182             else if (s[i] != ' ' && s[i] != '\t')
183                 return false;
184         }
185         return false;
186     }
187 
188     protected dstring commentLine(dstring s, int commentX) {
189         dchar[] res;
190         int x = 0;
191         bool commented = false;
192         foreach(i; 0 .. s.length) {
193             dchar ch = s[i];
194             if (ch == '\t') {
195                 int newX = (x + _content.tabSize) / _content.tabSize * _content.tabSize;
196                 if (!commented && newX >= commentX) {
197                     commented = true;
198                     if (newX != commentX) {
199                         // replace tab with space
200                         for (; x <= commentX; x++)
201                             res ~= ' ';
202                     } else {
203                         res ~= ch;
204                         x = newX;
205                     }
206                     res ~= "//"d;
207                     x += 2;
208                 } else {
209                     res ~= ch;
210                     x = newX;
211                 }
212             } else {
213                 if (!commented && x == commentX) {
214                     commented = true;
215                     res ~= "//"d;
216                     res ~= ch;
217                     x += 3;
218                 } else {
219                     res ~= ch;
220                     x++;
221                 }
222             }
223         }
224         if (!commented) {
225             for (; x < commentX; x++)
226                 res ~= ' ';
227             res ~= "//"d;
228         }
229         return cast(dstring)res;
230     }
231 
232     /// remove single line comment from beginning of line
233     protected dstring uncommentLine(dstring s) {
234         int p = -1;
235         foreach(int i; 0 .. cast(int)s.length - 1) {
236             if (s[i] == '/' && s[i + 1] == '/') {
237                 p = i;
238                 break;
239             }
240         }
241         if (p < 0)
242             return s;
243         s = s[0..p] ~ s[p + 2 .. $];
244         foreach(i; 0 .. s.length) {
245             if (s[i] != ' ' && s[i] != '\t') {
246                 return s;
247             }
248         }
249         return null;
250     }
251 
252     /// searches for neares token start before or equal to position
253     protected TextPosition tokenStart(TextPosition pos) {
254         TextPosition p = pos;
255         for (;;) {
256             TextPosition prevPos = content.prevCharPos(p);
257             if (p == prevPos)
258                 return p; // begin of file
259             TokenProp prop = content.tokenProp(p);
260             TokenProp prevProp = content.tokenProp(prevPos);
261             if (prop && prop != prevProp)
262                 return p;
263             p = prevPos;
264         }
265     }
266 
267     static struct TokenWithRange {
268         Token token;
269         TextRange range;
270         @property string toString() const {
271             return token.toString ~ range.toString;
272         }
273     }
274 
275     protected Token[] _tokens;
276     protected int _tokenIndex;
277 
278     protected bool initTokenizer() {
279         _tokens = tokenizeML(content.lines);
280         _tokenIndex = 0;
281         return true;
282     }
283 
284     protected TokenWithRange nextToken() {
285         TokenWithRange res;
286         if (_tokenIndex < _tokens.length) {
287             res.range.start = TextPosition(_tokens[_tokenIndex].line, _tokens[_tokenIndex].pos);
288             if (_tokenIndex + 1 < _tokens.length)
289                 res.range.end = TextPosition(_tokens[_tokenIndex + 1].line, _tokens[_tokenIndex + 1].pos);
290             else
291                 res.range.end = content.endOfFile();
292             res.token = _tokens[_tokenIndex];
293             _tokenIndex++;
294         } else {
295             res.range.end = res.range.start = content.endOfFile();
296         }
297         return res;
298     }
299 
300     protected TokenWithRange getPositionToken(TextPosition pos) {
301         initTokenizer();
302         for (;;) {
303             TokenWithRange tokenRange = nextToken();
304             //Log.d("read token: ", tokenRange);
305             if (tokenRange.token.type == TokenType.eof) {
306                 //Log.d("end of file");
307                 return tokenRange;
308             }
309             if (pos >= tokenRange.range.start && pos < tokenRange.range.end) {
310                 //Log.d("found: ", pos, " in ", tokenRange);
311                 return tokenRange;
312             }
313         }
314     }
315 
316     protected TokenWithRange[] getRangeTokens(TextRange range) {
317         TokenWithRange[] res;
318         initTokenizer();
319         for (;;) {
320             TokenWithRange tokenRange = nextToken();
321             //Log.d("read token: ", tokenRange);
322             if (tokenRange.token.type == TokenType.eof) {
323                 return res;
324             }
325             if (tokenRange.range.intersects(range)) {
326                 //Log.d("found: ", pos, " in ", tokenRange);
327                 res ~= tokenRange;
328             }
329         }
330     }
331 
332     protected bool isInsideBlockComment(TextPosition pos) {
333         TokenWithRange tokenRange = getPositionToken(pos);
334         if (tokenRange.token.type == TokenType.comment && tokenRange.token.isMultilineComment)
335             return pos > tokenRange.range.start && pos < tokenRange.range.end;
336         return false;
337     }
338 
339     /// toggle line comments for specified text range
340     override void toggleLineComment(TextRange range, Object source) {
341         TextRange r = content.fullLinesRange(range);
342         if (isInsideBlockComment(r.start) || isInsideBlockComment(r.end))
343             return;
344         int lineCount = r.end.line - r.start.line;
345         bool noEolAtEndOfRange = false;
346         if (lineCount == 0 || r.end.pos > 0) {
347             noEolAtEndOfRange = true;
348             lineCount++;
349         }
350         int minLeftX = -1;
351         bool hasComments = false;
352         bool hasNoComments = false;
353         bool hasNonEmpty = false;
354         dstring[] srctext;
355         dstring[] dsttext;
356         foreach(i; 0 .. lineCount) {
357             int lineIndex = r.start.line + i;
358             dstring s = content.line(lineIndex);
359             srctext ~= s;
360             TextLineMeasure m = content.measureLine(lineIndex);
361             if (!m.empty) {
362                 if (minLeftX < 0 || minLeftX > m.firstNonSpaceX)
363                     minLeftX = m.firstNonSpaceX;
364                 hasNonEmpty = true;
365                 if (isLineComment(s))
366                     hasComments = true;
367                 else
368                     hasNoComments = true;
369             }
370         }
371         if (minLeftX < 0)
372             minLeftX = 0;
373         if (hasNoComments || !hasComments) {
374             // comment
375             foreach(i; 0 .. lineCount) {
376                 dsttext ~= commentLine(srctext[i], minLeftX);
377             }
378             if (!noEolAtEndOfRange)
379                 dsttext ~= ""d;
380             EditOperation op = new EditOperation(EditAction.Replace, r, dsttext);
381             _content.performOperation(op, source);
382         } else {
383             // uncomment
384             foreach(i; 0 .. lineCount) {
385                 dsttext ~= uncommentLine(srctext[i]);
386             }
387             if (!noEolAtEndOfRange)
388                 dsttext ~= ""d;
389             EditOperation op = new EditOperation(EditAction.Replace, r, dsttext);
390             _content.performOperation(op, source);
391         }
392     }
393 
394     /// return true if toggle block comment is supported for file type
395     override @property bool supportsToggleBlockComment() {
396         return true;
397     }
398     /// return true if can toggle block comments for specified text range
399     override bool canToggleBlockComment(TextRange range) {
400         TokenWithRange startToken = getPositionToken(range.start);
401         TokenWithRange endToken = getPositionToken(range.end);
402         //Log.d("canToggleBlockComment: startToken=", startToken, " endToken=", endToken);
403         if (startToken.range == endToken.range && startToken.token.isMultilineComment) {
404             //Log.d("canToggleBlockComment: can uncomment");
405             return true;
406         }
407         if (range.empty)
408             return false;
409         TokenWithRange[] tokens = getRangeTokens(range);
410         foreach(ref t; tokens) {
411             if (t.token.type == TokenType.comment) {
412                 if (t.token.isMultilineComment) {
413                     // disable until nested comments support is implemented
414                     return false;
415                 } else {
416                     // single line comment
417                     if (t.range.isInside(range.start) || t.range.isInside(range.end))
418                         return false;
419                 }
420             }
421         }
422         return true;
423     }
424     /// toggle block comments for specified text range
425     override void toggleBlockComment(TextRange srcrange, Object source) {
426         TokenWithRange startToken = getPositionToken(srcrange.start);
427         TokenWithRange endToken = getPositionToken(srcrange.end);
428         if (startToken.range == endToken.range && startToken.token.isMultilineComment) {
429             TextRange range = startToken.range;
430             dstring[] dsttext;
431             foreach(i; range.start.line .. range.end.line + 1) {
432                 dstring s = content.line(i);
433                 int charsRemoved = 0;
434                 if (i == range.start.line) {
435                     int maxp = content.lineLength(range.start.line);
436                     if (i == range.end.line)
437                         maxp = range.end.pos - 2;
438                     charsRemoved = 2;
439                     foreach(j; range.start.pos + charsRemoved .. maxp) {
440                         if (s[j] != s[j - 1])
441                             break;
442                         charsRemoved++;
443                     }
444                     //Log.d("line before removing start of comment:", s);
445                     s = s[range.start.pos + charsRemoved .. $];
446                     //Log.d("line after removing start of comment:", s);
447                     charsRemoved += range.start.pos;
448                 }
449                 if (i == range.end.line) {
450                     int endp = range.end.pos;
451                     if (charsRemoved > 0)
452                         endp -= charsRemoved;
453                     int endRemoved = 2;
454                     for (int j = endp - endRemoved; j >= 0; j--) {
455                         if (s[j] != s[j + 1])
456                             break;
457                         endRemoved++;
458                     }
459                     //Log.d("line before removing end of comment:", s);
460                     s = s[0 .. endp - endRemoved];
461                     //Log.d("line after removing end of comment:", s);
462                 }
463                 dsttext ~= s;
464             }
465             EditOperation op = new EditOperation(EditAction.Replace, range, dsttext);
466             _content.performOperation(op, source);
467             return;
468         } else {
469             if (srcrange.empty)
470                 return;
471             TokenWithRange[] tokens = getRangeTokens(srcrange);
472             foreach(ref t; tokens) {
473                 if (t.token.type == TokenType.comment) {
474                     if (t.token.isMultilineComment) {
475                         // disable until nested comments support is implemented
476                         return;
477                     } else {
478                         // single line comment
479                         if (t.range.isInside(srcrange.start) || t.range.isInside(srcrange.end))
480                             return;
481                     }
482                 }
483             }
484             dstring[] dsttext;
485             foreach(i; srcrange.start.line .. srcrange.end.line + 1) {
486                 dstring s = content.line(i);
487                 int charsAdded = 0;
488                 if (i == srcrange.start.line) {
489                     int p = srcrange.start.pos;
490                     if (p < s.length) {
491                         s = s[p .. $];
492                         charsAdded = -p;
493                     } else {
494                         charsAdded = -(cast(int)s.length);
495                         s = null;
496                     }
497                     s = "/*" ~ s;
498                     charsAdded += 2;
499                 }
500                 if (i == srcrange.end.line) {
501                     int p = srcrange.end.pos + charsAdded;
502                     s = p > 0 ? s[0..p] : null;
503                     s ~= "*/";
504                 }
505                 dsttext ~= s;
506             }
507             EditOperation op = new EditOperation(EditAction.Replace, srcrange, dsttext);
508             _content.performOperation(op, source);
509             return;
510         }
511 
512     }
513 
514     /// categorize characters in content by token types
515     void updateHighlight(dstring[] lines, TokenPropString[] props, int changeStartLine, int changeEndLine) {
516 
517         initTokenizer();
518         _props = props;
519         changeStartLine = 0;
520         changeEndLine = cast(int)lines.length;
521         int tokenPos = 0;
522         int tokenLine = 0;
523         ubyte category = 0;
524         try {
525             for (;;) {
526                 TokenWithRange token = nextToken();
527                 if (token.token.type == TokenType.eof) {
528                     break;
529                 }
530                 uint newPos = token.range.start.pos;
531                 uint newLine = token.range.start.line;
532 
533                 // fill with category
534                 foreach(int i; tokenLine .. newLine + 1) {
535                     int start = i > tokenLine ? 0 : tokenPos;
536                     int end = i < newLine ? cast(int)lines[i].length : newPos;
537                     foreach(j; start .. end) {
538                         if (j < _props[i].length) {
539                             _props[i][j] = category;
540                         }
541                     }
542                 }
543 
544                 // handle token - convert to category
545                 switch(token.token.type) with(TokenType)
546                 {
547                     case comment:
548                         category = TokenCategory.Comment;
549                         break;
550                     case ident:
551                         if (isWidgetClassName(token.token.text))
552                             category = TokenCategory.Identifier_Class;
553                         else
554                             category = TokenCategory.Identifier;
555                         break;
556                     case str:
557                         category = TokenCategory.String;
558                         break;
559                     case integer:
560                         category = TokenCategory.Integer;
561                         break;
562                     case floating:
563                         category = TokenCategory.Float;
564                         break;
565                     case error:
566                         category = TokenCategory.Error;
567                         break;
568                     default:
569                         if (token.token.type >= colon)
570                             category = TokenCategory.Op;
571                         else
572                             category = 0;
573                         break;
574                 }
575                 tokenPos = newPos;
576                 tokenLine= newLine;
577 
578             }
579         } catch (Exception e) {
580             Log.e("exception while trying to parse DML source", e);
581         }
582         _props = null;
583     }
584 
585 
586     /// returns true if smart indent is supported
587     override bool supportsSmartIndents() {
588         return true;
589     }
590 
591     protected bool _opInProgress;
592     protected void applyNewLineSmartIndent(EditOperation op, Object source) {
593         int line = op.newRange.end.line;
594         if (line == 0)
595             return; // not for first line
596         int prevLine = line - 1;
597         TextLineMeasure lineMeasurement = _content.measureLine(line);
598         TextLineMeasure prevLineMeasurement = _content.measureLine(prevLine);
599         while (prevLineMeasurement.empty && prevLine > 0) {
600             prevLine--;
601             prevLineMeasurement = _content.measureLine(prevLine);
602         }
603         if (lineMeasurement.firstNonSpaceX >= 0 && lineMeasurement.firstNonSpaceX < prevLineMeasurement.firstNonSpaceX) {
604             dstring prevLineText = _content.line(prevLine);
605             TokenPropString prevLineTokenProps = _content.lineTokenProps(prevLine);
606             dchar lastOpChar = 0;
607             for (int j = prevLineMeasurement.lastNonSpace; j >= 0; j--) {
608                 auto cat = j < prevLineTokenProps.length ? tokenCategory(prevLineTokenProps[j]) : 0;
609                 if (cat == TokenCategory.Op) {
610                     lastOpChar = prevLineText[j];
611                     break;
612                 } else if (cat != TokenCategory.Comment && cat != TokenCategory.WhiteSpace) {
613                     break;
614                 }
615             }
616             int spacex = prevLineMeasurement.firstNonSpaceX;
617             if (lastOpChar == '{')
618                 spacex = _content.nextTab(spacex);
619             dstring txt = _content.fillSpace(spacex);
620             EditOperation op2 = new EditOperation(EditAction.Replace, TextRange(TextPosition(line, 0), TextPosition(line, lineMeasurement.firstNonSpace >= 0 ? lineMeasurement.firstNonSpace : 0)), [txt]);
621             _opInProgress = true;
622             _content.performOperation(op2, source);
623             _opInProgress = false;
624         }
625     }
626 
627     protected void applyClosingCurlySmartIndent(EditOperation op, Object source) {
628         int line = op.newRange.end.line;
629         TextPosition p2 = findPairedBracket(op.newRange.start);
630         if (p2 == op.newRange.start || p2.line > op.newRange.start.line)
631             return;
632         int prevLine = p2.line;
633         TextLineMeasure lineMeasurement = _content.measureLine(line);
634         TextLineMeasure prevLineMeasurement = _content.measureLine(prevLine);
635         if (lineMeasurement.firstNonSpace != op.newRange.start.pos)
636             return; // not in beginning of line
637         if (lineMeasurement.firstNonSpaceX >= 0 && lineMeasurement.firstNonSpaceX != prevLineMeasurement.firstNonSpaceX) {
638             int spacex = prevLineMeasurement.firstNonSpaceX;
639             if (spacex != lineMeasurement.firstNonSpaceX) {
640                 dstring txt = _content.fillSpace(spacex);
641                 txt = txt ~ "}";
642                 EditOperation op2 = new EditOperation(EditAction.Replace, TextRange(TextPosition(line, 0), TextPosition(line, lineMeasurement.firstNonSpace >= 0 ? lineMeasurement.firstNonSpace + 1 : 0)), [txt]);
643                 _opInProgress = true;
644                 _content.performOperation(op2, source);
645                 _opInProgress = false;
646             }
647         }
648     }
649 
650     /// apply smart indent, if supported
651     override void applySmartIndent(EditOperation op, Object source) {
652         if (_opInProgress)
653             return;
654         if (op.isInsertNewLine) {
655             // Enter key pressed - new line inserted or splitted
656             applyNewLineSmartIndent(op, source);
657         } else if (op.singleChar == '}') {
658             // } entered - probably need unindent
659             applyClosingCurlySmartIndent(op, source);
660         } else if (op.singleChar == '{') {
661             // { entered - probably need auto closing }
662         }
663     }
664 
665 }
666