Read a file as plain byte stream to find words that are not completely ASCII.
1794{
1795 GList* iconv_list = NULL, *conv_list = NULL, *iter;
1797 const gchar* enc;
1798 GHashTable* processed = NULL;
1799 gint n_impossible = 0;
1800 GError* error = NULL;
1801 gboolean clean_return = FALSE;
1802
1803 auto [file, thread] = try_gz_open (filename, "r",
1804 is_gzipped_file (filename), FALSE);
1805 if (file == NULL)
1806 {
1807 PWARN (
"Unable to open file %s", filename);
1808 goto cleanup_find_ambs;
1809 }
1810
1811
1813 ascii->encoding = g_quark_from_string ("ASCII");
1814 ascii->iconv = g_iconv_open ("UTF-8", "ASCII");
1815 if (ascii->iconv == (GIConv) - 1)
1816 {
1817 PWARN (
"Unable to open ASCII ICONV conversion descriptor");
1818 goto cleanup_find_ambs;
1819 }
1820
1821
1822 for (iter = encodings; iter; iter = iter->next)
1823 {
1825 iconv_item->encoding = GPOINTER_TO_UINT (iter->data);
1826 if (iconv_item->encoding == ascii->encoding)
1827 {
1828 continue;
1829 }
1830
1831 enc = g_quark_to_string (iconv_item->encoding);
1832 iconv_item->iconv = g_iconv_open ("UTF-8", enc);
1833 if (iconv_item->iconv == (GIConv) - 1)
1834 {
1835 PWARN (
"Unable to open IConv conversion descriptor for '%s'", enc);
1836 g_free (iconv_item);
1837 goto cleanup_find_ambs;
1838 }
1839 else
1840 {
1841 iconv_list = g_list_prepend (iconv_list, iconv_item);
1842 }
1843 }
1844
1845
1846 if (unique)
1847 *unique = g_hash_table_new_full (g_str_hash, g_str_equal, g_free,
1848 (GDestroyNotify) conv_free);
1849 if (ambiguous)
1850 *ambiguous = g_hash_table_new_full (g_str_hash, g_str_equal, g_free,
1851 (GDestroyNotify) conv_list_free);
1852 if (impossible)
1853 *impossible = NULL;
1854 processed = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
1855
1856
1857 while (1)
1858 {
1859 gchar line[256], *word, *utf8;
1860 gchar** word_array, **word_cursor;
1862
1863 if (!fgets (line, sizeof (line) - 1, file))
1864 {
1865 if (feof (file))
1866 {
1867 break;
1868 }
1869 else
1870 {
1871 goto cleanup_find_ambs;
1872 }
1873 }
1874
1875 g_strchomp (line);
1876 replace_character_references (line);
1877 word_array = g_strsplit_set (line, "> <", 0);
1878
1879
1880 for (word_cursor = word_array; *word_cursor; word_cursor++)
1881 {
1882 word = *word_cursor;
1883 if (!word)
1884 continue;
1885
1886 utf8 = g_convert_with_iconv (word, -1, ascii->iconv,
1887 NULL, NULL, &error);
1888 if (utf8)
1889 {
1890
1891 g_free (utf8);
1892 continue;
1893 }
1894 g_error_free (error);
1895 error = NULL;
1896
1897 if (g_hash_table_lookup_extended (processed, word, NULL, NULL))
1898 {
1899
1900 continue;
1901 }
1902
1903
1904 conv_list = NULL;
1905 for (iter = iconv_list; iter; iter = iter->next)
1906 {
1907 iconv_item = static_cast<decltype (iconv_item)> (iter->data);
1908 utf8 = g_convert_with_iconv (word, -1, iconv_item->iconv,
1909 NULL, NULL, &error);
1910 if (utf8)
1911 {
1913 conv->encoding = iconv_item->encoding;
1914 conv->utf8_string = utf8;
1915 conv_list = g_list_prepend (conv_list, conv);
1916 }
1917 else
1918 {
1919 g_error_free (error);
1920 error = NULL;
1921 }
1922 }
1923
1924
1925 if (!conv_list)
1926 {
1927 if (impossible)
1928 *impossible = g_list_append (*impossible, g_strdup (word));
1929 n_impossible++;
1930 }
1931
1932
1933 else if (conv_list->next)
1934 {
1935 if (ambiguous)
1936 {
1937 g_hash_table_insert (*ambiguous, g_strdup (word), conv_list);
1938 }
1939 else
1940 {
1941 conv_list_free (conv_list);
1942 }
1943 }
1944
1945
1946 else
1947 {
1948 if (unique)
1949 {
1950 g_hash_table_insert (*unique, g_strdup (word), conv);
1951 }
1952 else
1953 {
1954 conv_free (conv);
1955 }
1956 g_list_free (conv_list);
1957 }
1958
1959 g_hash_table_insert (processed, g_strdup (word), NULL);
1960 }
1961 g_strfreev (word_array);
1962 }
1963
1964 clean_return = TRUE;
1965
1966cleanup_find_ambs:
1967
1968 if (iconv_list)
1969 {
1970 for (iter = iconv_list; iter; iter = iter->next)
1971 {
1972 if (iter->data)
1973 {
1975 g_free (iter->data);
1976 }
1977 }
1978 g_list_free (iconv_list);
1979 }
1980 if (processed)
1981 g_hash_table_destroy (processed);
1982 if (ascii)
1983 g_free (ascii);
1984 if (file)
1985 {
1986 fclose (file);
1987 if (thread)
1988 g_thread_join (thread);
1989 }
1990
1991 return (clean_return) ? n_impossible : -1;
1992}
#define PWARN(format, args...)
Log a warning.