FLANG

token-sequence.h
1 //===-- lib/Parser/token-sequence.h -----------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef FORTRAN_PARSER_TOKEN_SEQUENCE_H_
10 #define FORTRAN_PARSER_TOKEN_SEQUENCE_H_
11
12 // A buffer class capable of holding a contiguous sequence of characters
13 // and a partitioning thereof into preprocessing tokens, along with their
14 // associated provenances.
15
16 #include "flang/Parser/char-block.h"
17 #include "flang/Parser/provenance.h"
18 #include <cstddef>
19 #include <cstring>
20 #include <string>
21 #include <utility>
22 #include <vector>
23
24 namespace llvm {
25 class raw_ostream;
26 }
27
28 namespace Fortran::parser {
29
30 // Buffers a contiguous sequence of characters that has been partitioned into
31 // a sequence of preprocessing tokens with provenances.
32 class TokenSequence {
33 public:
34 TokenSequence() {}
35 TokenSequence(const TokenSequence &that) { Put(that); }
36 TokenSequence(
37 const TokenSequence &that, std::size_t at, std::size_t count = 1) {
38 Put(that, at, count);
39 }
40 TokenSequence(TokenSequence &&that)
41 : start_{std::move(that.start_)}, nextStart_{that.nextStart_},
42 char_{std::move(that.char_)}, provenances_{
43 std::move(that.provenances_)} {}
44 TokenSequence(const std::string &s, Provenance p) { Put(s, p); }
45
46 TokenSequence &operator=(const TokenSequence &that) {
47 clear();
48 Put(that);
49 return *this;
50 }
51 TokenSequence &operator=(TokenSequence &&that);
52 bool empty() const { return start_.empty(); }
53 void clear();
54 void pop_back();
55 void shrink_to_fit();
56 void swap(TokenSequence &);
57
58 std::size_t SizeInTokens() const { return start_.size(); }
59 std::size_t SizeInChars() const { return char_.size(); }
60
61 CharBlock ToCharBlock() const { return {&char_[0], char_.size()}; }
62 std::string ToString() const { return ToCharBlock().ToString(); }
63
64 CharBlock TokenAt(std::size_t token) const {
65 return {&char_[start_.at(token)], TokenBytes(token)};
66 }
67 char CharAt(std::size_t j) const { return char_.at(j); }
68 CharBlock CurrentOpenToken() const {
69 return {&char_[nextStart_], char_.size() - nextStart_};
70 }
71
72 std::size_t SkipBlanks(std::size_t) const;
73
74 void PutNextTokenChar(char ch, Provenance provenance) {
75 char_.emplace_back(ch);
76 provenances_.Put({provenance, 1});
77 }
78
79 void CloseToken() {
80 start_.emplace_back(nextStart_);
81 nextStart_ = char_.size();
82 }
83
84 void ReopenLastToken() {
85 nextStart_ = start_.back();
86 start_.pop_back();
87 }
88
89 void RemoveLastToken();
90
91 void Put(const TokenSequence &);
92 void Put(const TokenSequence &, ProvenanceRange);
93 void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
94 void Put(const char *, std::size_t, Provenance);
95 void Put(const CharBlock &, Provenance);
96 void Put(const std::string &, Provenance);
97 void Put(llvm::raw_string_ostream &, Provenance);
98
99 Provenance GetTokenProvenance(
100 std::size_t token, std::size_t offset = 0) const;
101 ProvenanceRange GetTokenProvenanceRange(
102 std::size_t token, std::size_t offset = 0) const;
103 ProvenanceRange GetIntervalProvenanceRange(
104 std::size_t token, std::size_t tokens = 1) const;
105 ProvenanceRange GetProvenanceRange() const;
106
107 char *GetMutableCharData() { return &char_[0]; }
108 TokenSequence &ToLowerCase();
109 bool HasBlanks(std::size_t firstChar = 0) const;
110 bool HasRedundantBlanks(std::size_t firstChar = 0) const;
111 TokenSequence &RemoveBlanks(std::size_t firstChar = 0);
112 TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0);
113 TokenSequence &ClipComment(bool skipFirst = false);
114 void Emit(CookedSource &) const;
115 void Dump(llvm::raw_ostream &) const;
116
117 private:
118 std::size_t TokenBytes(std::size_t token) const {
119 return (token + 1 >= start_.size() ? char_.size() : start_[token + 1]) -
120 start_[token];
121 }
122
123 std::vector<std::size_t> start_;
124 std::size_t nextStart_{0};
125 std::vector<char> char_;
126 OffsetToProvenanceMappings provenances_;
127 };
128 } // namespace Fortran::parser
129 #endif // FORTRAN_PARSER_TOKEN_SEQUENCE_H_
Fortran::parser::CookedSource
Definition: provenance.h:216
Fortran::parser::Provenance
Definition: provenance.h:50
Fortran::parser::OffsetToProvenanceMappings
Definition: provenance.h:110
Fortran::parser::TokenSequence
Definition: token-sequence.h:32
Fortran::parser::CharBlock
Definition: char-block.h:28
Fortran::common::Interval< Provenance >
Fortran::parser
Definition: check-expression.h:19