Coverage for src / charset_normalizer / legacy.py: 83%

1from __future__ import annotations 

2 

3from typing import TYPE_CHECKING, Any 

4from warnings import warn 

5 

6from .api import from_bytes 

7from .constant import CHARDET_CORRESPONDENCE, TOO_SMALL_SEQUENCE 

8 

9if TYPE_CHECKING: 

10 from typing import TypedDict 

11 

12 class ResultDict(TypedDict): 

13 encoding: str | None 

14 language: str 

15 confidence: float | None 

16 

17 

18def detect( 

19 byte_str: bytes, should_rename_legacy: bool = False, **kwargs: Any 

20) -> ResultDict: 

21 """ 

22 chardet legacy method 

23 Detect the encoding of the given byte string. It should be mostly backward-compatible. 

24 Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it) 

25 This function is deprecated and should be used to migrate your project easily, consult the documentation for 

26 further information. Not planned for removal. 

27 

28 :param byte_str: The byte sequence to examine. 

29 :param should_rename_legacy: Should we rename legacy encodings 

30 to their more modern equivalents? 

31 """ 

32 if len(kwargs): 

33 warn( 

34 f"charset-normalizer disregard arguments '{','.join(list(kwargs.keys()))}' in legacy function detect()" 

35 ) 

36 

37 if not isinstance(byte_str, (bytearray, bytes)): 

38 raise TypeError( # pragma: nocover 

39 f"Expected object of type bytes or bytearray, got: {type(byte_str)}" 

40 ) 

41 

42 if isinstance(byte_str, bytearray): 

43 byte_str = bytes(byte_str) 

44 

45 r = from_bytes(byte_str).best() 

46 

47 encoding = r.encoding if r is not None else None 

48 language = r.language if r is not None and r.language != "Unknown" else "" 

49 confidence = 1.0 - r.chaos if r is not None else None 

50 

51 # automatically lower confidence 

52 # on small bytes samples. 

53 # https://github.com/jawah/charset_normalizer/issues/391 

54 if ( 

55 confidence is not None 

56 and confidence >= 0.9 

57 and encoding 

58 not in { 

59 "utf_8", 

60 "ascii", 

61 } 

62 and not r.bom # type: ignore[union-attr] 

63 and len(byte_str) < TOO_SMALL_SEQUENCE 

64 ): 

65 confidence -= 0.2 

66 

67 # Note: CharsetNormalizer does not return 'UTF-8-SIG' as the sig get stripped in the detection/normalization process 

68 # but chardet does return 'utf-8-sig' and it is a valid codec name. 

69 if r is not None and encoding == "utf_8" and r.bom: 

70 encoding += "_sig" 

71 

72 if not should_rename_legacy and encoding in CHARDET_CORRESPONDENCE: 

73 encoding = CHARDET_CORRESPONDENCE[encoding] 

74 

75 return { 

76 "encoding": encoding, 

77 "language": language, 

78 "confidence": confidence, 

79 }