FLANG

characters.h
1 //===-- include/flang/Parser/characters.h -----------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef FORTRAN_PARSER_CHARACTERS_H_
10 #define FORTRAN_PARSER_CHARACTERS_H_
11
12 // Define some character classification predicates and
13 // conversions here to avoid dependences upon <cctype> and
14 // also to accomodate Fortran tokenization.
15
16 #include <cstddef>
17 #include <optional>
18 #include <string>
19
20 namespace Fortran::parser {
21
22 extern bool useHexadecimalEscapeSequences;
23
24 // We can easily support Fortran program source in any character
25 // set whose first 128 code points correspond to ASCII codes 0-127 (ISO/IEC646).
26 // The specific encodings that we can handle include:
27 // LATIN_1: ISO 8859-1 Latin-1
28 // UTF_8: Multi-byte encoding of Unicode (ISO/IEC 10646)
29 enum class Encoding { LATIN_1, UTF_8 };
30
31 inline constexpr bool IsUpperCaseLetter(char ch) {
32 return ch >= 'A' && ch <= 'Z';
33 }
34
35 inline constexpr bool IsLowerCaseLetter(char ch) {
36 return ch >= 'a' && ch <= 'z';
37 }
38
39 inline constexpr bool IsLetter(char ch) {
40 return IsUpperCaseLetter(ch) || IsLowerCaseLetter(ch);
41 }
42
43 inline constexpr bool IsDecimalDigit(char ch) { return ch >= '0' && ch <= '9'; }
44
45 inline constexpr bool IsHexadecimalDigit(char ch) {
46 return (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') ||
47 (ch >= 'a' && ch <= 'f');
48 }
49
50 inline constexpr bool IsOctalDigit(char ch) { return ch >= '0' && ch <= '7'; }
51
52 inline constexpr bool IsLegalIdentifierStart(char ch) {
53 return IsLetter(ch) || ch == '_' || ch == '@' || ch == '$';
54 }
55
56 inline constexpr bool IsLegalInIdentifier(char ch) {
57 return IsLegalIdentifierStart(ch) || IsDecimalDigit(ch);
58 }
59
60 inline constexpr char ToLowerCaseLetter(char ch) {
61 return IsUpperCaseLetter(ch) ? ch - 'A' + 'a' : ch;
62 }
63
64 inline constexpr char ToLowerCaseLetter(char &&ch) {
65 return IsUpperCaseLetter(ch) ? ch - 'A' + 'a' : ch;
66 }
67
68 inline std::string ToLowerCaseLetters(const std::string &str) {
69 std::string lowered{str};
70 for (char &ch : lowered) {
71 ch = ToLowerCaseLetter(ch);
72 }
73 return lowered;
74 }
75
76 inline constexpr char ToUpperCaseLetter(char ch) {
77 return IsLowerCaseLetter(ch) ? ch - 'a' + 'A' : ch;
78 }
79
80 inline constexpr char ToUpperCaseLetter(char &&ch) {
81 return IsLowerCaseLetter(ch) ? ch - 'a' + 'A' : ch;
82 }
83
84 inline std::string ToUpperCaseLetters(const std::string &str) {
85 std::string raised{str};
86 for (char &ch : raised) {
87 ch = ToUpperCaseLetter(ch);
88 }
89 return raised;
90 }
91
92 inline constexpr bool IsSameApartFromCase(char x, char y) {
93 return ToLowerCaseLetter(x) == ToLowerCaseLetter(y);
94 }
95
96 inline constexpr char DecimalDigitValue(char ch) { return ch - '0'; }
97
98 inline constexpr char HexadecimalDigitValue(char ch) {
99 return IsUpperCaseLetter(ch) ? ch - 'A' + 10
100 : IsLowerCaseLetter(ch) ? ch - 'a' + 10
101 : DecimalDigitValue(ch);
102 }
103
104 inline constexpr std::optional<char> BackslashEscapeValue(char ch) {
105 switch (ch) {
106 case 'a':
107 return std::nullopt; // '\a'; PGF90 doesn't know \a
108 case 'b':
109 return '\b';
110 case 'f':
111 return '\f';
112 case 'n':
113 return '\n';
114 case 'r':
115 return '\r';
116 case 't':
117 return '\t';
118 case 'v':
119 return '\v';
120 case '"':
121 case '\'':
122 case '\\':
123 return ch;
124 default:
125 return std::nullopt;
126 }
127 }
128
129 inline constexpr std::optional<char> BackslashEscapeChar(char ch) {
130 switch (ch) {
131 case '\a':
132 return std::nullopt; // 'a'; PGF90 doesn't know \a
133 case '\b':
134 return 'b';
135 case '\f':
136 return 'f';
137 case '\n':
138 return 'n';
139 case '\r':
140 return 'r';
141 case '\t':
142 return 't';
143 case '\v':
144 return 'v';
145 case '"':
146 case '\'':
147 case '\\':
148 return ch;
149 default:
150 return std::nullopt;
151 }
152 }
153
154 struct EncodedCharacter {
155 static constexpr int maxEncodingBytes{6};
156 char buffer[maxEncodingBytes];
157 int bytes{0};
158 };
159
160 template <Encoding ENCODING> EncodedCharacter EncodeCharacter(char32_t ucs);
161 template <> EncodedCharacter EncodeCharacter<Encoding::LATIN_1>(char32_t);
162 template <> EncodedCharacter EncodeCharacter<Encoding::UTF_8>(char32_t);
163
164 EncodedCharacter EncodeCharacter(Encoding, char32_t ucs);
165
166 template <Encoding ENCODING, typename STRING>
167 std::string EncodeString(const STRING &);
168 extern template std::string EncodeString<Encoding::LATIN_1, std::string>(
169 const std::string &);
170 extern template std::string EncodeString<Encoding::UTF_8, std::u32string>(
171 const std::u32string &);
172
173 // EmitQuotedChar drives callbacks "emit" and "insert" to output the
174 // bytes of an encoding for a codepoint.
175 template <typename NORMAL, typename INSERTED>
176 void EmitQuotedChar(char32_t ch, const NORMAL &emit, const INSERTED &insert,
177 bool backslashEscapes = true, Encoding encoding = Encoding::UTF_8) {
178 auto emitOneByte{[&](std::uint8_t ch) {
179 if (backslashEscapes && (ch < ' ' || ch >= 0x7f || ch == '\\')) {
180 if (std::optional<char> escape{BackslashEscapeChar(ch)}) {
181 insert('\\');
182 emit(*escape);
183 } else if (useHexadecimalEscapeSequences) {
184 insert('\\');
185 insert('x');
186 int top{ch >> 4}, bottom{ch & 0xf};
187 insert(top > 9 ? 'a' + top - 10 : '0' + top);
188 insert(bottom > 9 ? 'a' + bottom - 10 : '0' + bottom);
189 } else {
190 // octal escape sequence; always emit 3 digits to avoid ambiguity
191 insert('\\');
192 insert('0' + (ch >> 6));
193 insert('0' + ((ch >> 3) & 7));
194 insert('0' + (ch & 7));
195 }
196 } else if (ch == '\n') { // always escape newlines
197 insert('\\');
198 insert('n');
199 } else {
200 emit(ch);
201 }
202 }};
203 if (ch <= 0x7f) {
204 emitOneByte(ch);
205 } else {
206 EncodedCharacter encoded{EncodeCharacter(encoding, ch)};
207 for (int j{0}; j < encoded.bytes; ++j) {
208 emitOneByte(encoded.buffer[j]);
209 }
210 }
211 }
212
213 std::string QuoteCharacterLiteral(const std::string &,
214 bool backslashEscapes = true, Encoding = Encoding::LATIN_1);
215 std::string QuoteCharacterLiteral(const std::u16string &,
216 bool backslashEscapes = true, Encoding = Encoding::UTF_8);
217 std::string QuoteCharacterLiteral(const std::u32string &,
218 bool backslashEscapes = true, Encoding = Encoding::UTF_8);
219
220 int UTF_8CharacterBytes(const char *);
221
222 struct DecodedCharacter {
223 char32_t codepoint{0};
224 int bytes{0}; // signifying failure
225 };
226
227 template <Encoding ENCODING>
228 DecodedCharacter DecodeRawCharacter(const char *, std::size_t);
229 template <>
230 DecodedCharacter DecodeRawCharacter<Encoding::LATIN_1>(
231 const char *, std::size_t);
232
233 template <>
234 DecodedCharacter DecodeRawCharacter<Encoding::UTF_8>(const char *, std::size_t);
235
236 // DecodeCharacter optionally handles backslash escape sequences, too.
237 template <Encoding ENCODING>
238 DecodedCharacter DecodeCharacter(
239 const char *, std::size_t, bool backslashEscapes);
240 extern template DecodedCharacter DecodeCharacter<Encoding::LATIN_1>(
241 const char *, std::size_t, bool);
242 extern template DecodedCharacter DecodeCharacter<Encoding::UTF_8>(
243 const char *, std::size_t, bool);
244
245 DecodedCharacter DecodeCharacter(
246 Encoding, const char *, std::size_t, bool backslashEscapes);
247
248 template <typename RESULT, Encoding ENCODING>
249 RESULT DecodeString(const std::string &, bool backslashEscapes);
250 extern template std::string DecodeString<std::string, Encoding::LATIN_1>(
251 const std::string &, bool);
252 extern template std::u16string DecodeString<std::u16string, Encoding::UTF_8>(
253 const std::string &, bool);
254 extern template std::u32string DecodeString<std::u32string, Encoding::UTF_8>(
255 const std::string &, bool);
256 } // namespace Fortran::parser
257 #endif // FORTRAN_PARSER_CHARACTERS_H_
Fortran::parser::EncodedCharacter
Definition: characters.h:154
Fortran::parser
Definition: check-expression.h:19
Fortran::parser::DecodedCharacter
Definition: characters.h:222