Coverage for src / charset_normalizer / cli / __main__.py: 80%
96 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-11-01 13:35 +0000
1from __future__ import annotations
3import argparse
4import sys
5import typing
6from os.path import abspath, basename, dirname, join, realpath
7from platform import python_version
8from unicodedata import unidata_version
10import charset_normalizer.md as md_module
11from charset_normalizer import from_fp
12from charset_normalizer.models import CliDetectionResult
13from charset_normalizer.version import __version__
16def query_yes_no(question: str, default: str = "yes") -> bool: # Defensive:
17 """Ask a yes/no question via input() and return the answer as a bool."""
18 prompt = " [Y/n] " if default == "yes" else " [y/N] "
20 while True:
21 choice = input(question + prompt).strip().lower()
22 if not choice:
23 return default == "yes"
24 if choice in ("y", "yes"):
25 return True
26 if choice in ("n", "no"):
27 return False
28 print("Please respond with 'y' or 'n'.")
31class FileType:
32 """Factory for creating file object types
34 Instances of FileType are typically passed as type= arguments to the
35 ArgumentParser add_argument() method.
37 Keyword Arguments:
38 - mode -- A string indicating how the file is to be opened. Accepts the
39 same values as the builtin open() function.
40 - bufsize -- The file's desired buffer size. Accepts the same values as
41 the builtin open() function.
42 - encoding -- The file's encoding. Accepts the same values as the
43 builtin open() function.
44 - errors -- A string indicating how encoding and decoding errors are to
45 be handled. Accepts the same value as the builtin open() function.
47 Backported from CPython 3.12
48 """
50 def __init__(
51 self,
52 mode: str = "r",
53 bufsize: int = -1,
54 encoding: str | None = None,
55 errors: str | None = None,
56 ):
57 self._mode = mode
58 self._bufsize = bufsize
59 self._encoding = encoding
60 self._errors = errors
62 def __call__(self, string: str) -> typing.IO: # type: ignore[type-arg]
63 # the special argument "-" means sys.std{in,out}
64 if string == "-":
65 if "r" in self._mode:
66 return sys.stdin.buffer if "b" in self._mode else sys.stdin
67 elif any(c in self._mode for c in "wax"):
68 return sys.stdout.buffer if "b" in self._mode else sys.stdout
69 else:
70 msg = f'argument "-" with mode {self._mode}'
71 raise ValueError(msg)
73 # all other arguments are used as file names
74 try:
75 return open(string, self._mode, self._bufsize, self._encoding, self._errors)
76 except OSError as e:
77 message = f"can't open '{string}': {e}"
78 raise argparse.ArgumentTypeError(message)
80 def __repr__(self) -> str:
81 args = self._mode, self._bufsize
82 kwargs = [("encoding", self._encoding), ("errors", self._errors)]
83 args_str = ", ".join(
84 [repr(arg) for arg in args if arg != -1]
85 + [f"{kw}={arg!r}" for kw, arg in kwargs if arg is not None]
86 )
87 return f"{type(self).__name__}({args_str})"
90def cli_detect(argv: list[str] | None = None) -> int:
91 """
92 CLI assistant using ARGV and ArgumentParser
93 :param argv:
94 :return: 0 if everything is fine, anything else equal trouble
95 """
96 parser = argparse.ArgumentParser(
97 description="The Real First Universal Charset Detector. "
98 "Discover originating encoding used on text file. "
99 "Normalize text to unicode."
100 )
102 parser.add_argument(
103 "files", type=FileType("rb"), nargs="+", help="File(s) to be analysed"
104 )
105 parser.add_argument(
106 "-v",
107 "--verbose",
108 action="store_true",
109 default=False,
110 dest="verbose",
111 help="Display complementary information about file if any. "
112 "Stdout will contain logs about the detection process.",
113 )
114 parser.add_argument(
115 "-a",
116 "--with-alternative",
117 action="store_true",
118 default=False,
119 dest="alternatives",
120 help="Output complementary possibilities if any. Top-level JSON WILL be a list.",
121 )
122 parser.add_argument(
123 "-n",
124 "--normalize",
125 action="store_true",
126 default=False,
127 dest="normalize",
128 help="Permit to normalize input file. If not set, program does not write anything.",
129 )
130 parser.add_argument(
131 "-m",
132 "--minimal",
133 action="store_true",
134 default=False,
135 dest="minimal",
136 help="Only output the charset detected to STDOUT. Disabling JSON output.",
137 )
138 parser.add_argument(
139 "-r",
140 "--replace",
141 action="store_true",
142 default=False,
143 dest="replace",
144 help="Replace file when trying to normalize it instead of creating a new one.",
145 )
146 parser.add_argument(
147 "-f",
148 "--force",
149 action="store_true",
150 default=False,
151 dest="force",
152 help="Replace file without asking if you are sure, use this flag with caution.",
153 )
154 parser.add_argument(
155 "-i",
156 "--no-preemptive",
157 action="store_true",
158 default=False,
159 dest="no_preemptive",
160 help="Disable looking at a charset declaration to hint the detector.",
161 )
162 parser.add_argument(
163 "-t",
164 "--threshold",
165 action="store",
166 default=0.2,
167 type=float,
168 dest="threshold",
169 help="Define a custom maximum amount of noise allowed in decoded content. 0. <= noise <= 1.",
170 )
171 parser.add_argument(
172 "--version",
173 action="version",
174 version="Charset-Normalizer {} - Python {} - Unicode {} - SpeedUp {}".format(
175 __version__,
176 python_version(),
177 unidata_version,
178 "OFF" if md_module.__file__.lower().endswith(".py") else "ON",
179 ),
180 help="Show version information and exit.",
181 )
183 args = parser.parse_args(argv)
185 if args.replace is True and args.normalize is False:
186 if args.files:
187 for my_file in args.files:
188 my_file.close()
189 print("Use --replace in addition of --normalize only.", file=sys.stderr)
190 return 1
192 if args.force is True and args.replace is False:
193 if args.files:
194 for my_file in args.files:
195 my_file.close()
196 print("Use --force in addition of --replace only.", file=sys.stderr)
197 return 1
199 if args.threshold < 0.0 or args.threshold > 1.0:
200 if args.files:
201 for my_file in args.files:
202 my_file.close()
203 print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr)
204 return 1
206 x_ = []
208 for my_file in args.files:
209 matches = from_fp(
210 my_file,
211 threshold=args.threshold,
212 explain=args.verbose,
213 preemptive_behaviour=args.no_preemptive is False,
214 )
216 best_guess = matches.best()
218 if best_guess is None:
219 print(
220 'Unable to identify originating encoding for "{}". {}'.format(
221 my_file.name,
222 (
223 "Maybe try increasing maximum amount of chaos."
224 if args.threshold < 1.0
225 else ""
226 ),
227 ),
228 file=sys.stderr,
229 )
230 x_.append(
231 CliDetectionResult(
232 abspath(my_file.name),
233 None,
234 [],
235 [],
236 "Unknown",
237 [],
238 False,
239 1.0,
240 0.0,
241 None,
242 True,
243 )
244 )
245 else:
246 cli_result = CliDetectionResult(
247 abspath(my_file.name),
248 best_guess.encoding,
249 best_guess.encoding_aliases,
250 [
251 cp
252 for cp in best_guess.could_be_from_charset
253 if cp != best_guess.encoding
254 ],
255 best_guess.language,
256 best_guess.alphabets,
257 best_guess.bom,
258 best_guess.percent_chaos,
259 best_guess.percent_coherence,
260 None,
261 True,
262 )
263 x_.append(cli_result)
265 if len(matches) > 1 and args.alternatives:
266 for el in matches:
267 if el != best_guess:
268 x_.append(
269 CliDetectionResult(
270 abspath(my_file.name),
271 el.encoding,
272 el.encoding_aliases,
273 [
274 cp
275 for cp in el.could_be_from_charset
276 if cp != el.encoding
277 ],
278 el.language,
279 el.alphabets,
280 el.bom,
281 el.percent_chaos,
282 el.percent_coherence,
283 None,
284 False,
285 )
286 )
288 if args.normalize is True:
289 if best_guess.encoding.startswith("utf") is True:
290 print(
291 '"{}" file does not need to be normalized, as it already came from unicode.'.format(
292 my_file.name
293 ),
294 file=sys.stderr,
295 )
296 if my_file.closed is False:
297 my_file.close()
298 continue
300 dir_path = dirname(realpath(my_file.name))
301 file_name = basename(realpath(my_file.name))
303 o_: list[str] = file_name.split(".")
305 if args.replace is False:
306 o_.insert(-1, best_guess.encoding)
307 if my_file.closed is False:
308 my_file.close()
309 elif (
310 args.force is False
311 and query_yes_no(
312 'Are you sure to normalize "{}" by replacing it ?'.format(
313 my_file.name
314 ),
315 "no",
316 )
317 is False
318 ):
319 if my_file.closed is False:
320 my_file.close()
321 continue
323 try:
324 cli_result.unicode_path = join(dir_path, ".".join(o_))
326 with open(cli_result.unicode_path, "wb") as fp:
327 fp.write(best_guess.output())
328 except OSError as e: # Defensive:
329 print(str(e), file=sys.stderr)
330 if my_file.closed is False:
331 my_file.close()
332 return 2
334 if my_file.closed is False:
335 my_file.close()
337 if args.minimal is False:
338 from json import dumps
340 print(
341 dumps(
342 [el.__dict__ for el in x_]
343 if args.alternatives or len(x_) > 1
344 else x_[0].__dict__,
345 ensure_ascii=True,
346 indent=4,
347 )
348 )
349 else:
350 for my_file in args.files:
351 print(
352 ", ".join(
353 [
354 el.encoding or "undefined"
355 for el in x_
356 if el.path == abspath(my_file.name)
357 ]
358 )
359 )
361 return 0
364if __name__ == "__main__": # Defensive:
365 raise SystemExit(cli_detect())