1# 2# This class knows how to package up [e]glibc. Its shared since prebuild binary toolchains 3# may need packaging and its pointless to duplicate this code. 4# 5# Caller should set GLIBC_INTERNAL_USE_BINARY_LOCALE to one of: 6# "compile" - Use QEMU to generate the binary locale files 7# "precompiled" - The binary locale files are pregenerated and already present 8# "ondevice" - The device will build the locale files upon first boot through the postinst 9 10GLIBC_INTERNAL_USE_BINARY_LOCALE ?= "ondevice" 11 12GLIBC_SPLIT_LC_PACKAGES ?= "0" 13 14python __anonymous () { 15 enabled = d.getVar("ENABLE_BINARY_LOCALE_GENERATION") 16 17 pn = d.getVar("PN") 18 if pn.endswith("-initial"): 19 enabled = False 20 21 if enabled and int(enabled): 22 import re 23 24 target_arch = d.getVar("TARGET_ARCH") 25 binary_arches = d.getVar("BINARY_LOCALE_ARCHES") or "" 26 use_cross_localedef = d.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF") or "" 27 28 for regexp in binary_arches.split(" "): 29 r = re.compile(regexp) 30 31 if r.match(target_arch): 32 depends = d.getVar("DEPENDS") 33 if use_cross_localedef == "1" : 34 depends = "%s cross-localedef-native" % depends 35 else: 36 depends = "%s qemu-native" % depends 37 d.setVar("DEPENDS", depends) 38 d.setVar("GLIBC_INTERNAL_USE_BINARY_LOCALE", "compile") 39 break 40} 41 42# try to fix disable charsets/locales/locale-code compile fail 43PACKAGE_NO_GCONV ?= "0" 44 45OVERRIDES:append = ":${TARGET_ARCH}-${TARGET_OS}" 46 47locale_base_postinst_ontarget() { 48mkdir ${libdir}/locale 49localedef --inputfile=${datadir}/i18n/locales/%s --charmap=%s %s 50} 51 52locale_base_postrm() { 53#!/bin/sh 54localedef --delete-from-archive --inputfile=${datadir}/locales/%s --charmap=%s %s 55} 56 57LOCALETREESRC ?= "${PKGD}" 58 59do_prep_locale_tree() { 60 treedir=${WORKDIR}/locale-tree 61 rm -rf $treedir 62 mkdir -p $treedir/${base_bindir} $treedir/${base_libdir} $treedir/${datadir} $treedir/${localedir} 63 tar -cf - -C ${LOCALETREESRC}${datadir} -p i18n | tar -xf - -C $treedir/${datadir} 64 # unzip to avoid parsing errors 65 for i in $treedir/${datadir}/i18n/charmaps/*gz; do 66 gunzip $i 67 done 68 # The extract pattern "./l*.so*" is carefully selected so that it will 69 # match ld*.so and lib*.so*, but not any files in the gconv directory 70 # (if it exists). This makes sure we only unpack the files we need. 71 # This is important in case usrmerge is set in DISTRO_FEATURES, which 72 # means ${base_libdir} == ${libdir}. 73 tar -cf - -C ${LOCALETREESRC}${base_libdir} -p . | tar -xf - -C $treedir/${base_libdir} --wildcards './l*.so*' 74 if [ -f ${STAGING_LIBDIR_NATIVE}/libgcc_s.* ]; then 75 tar -cf - -C ${STAGING_LIBDIR_NATIVE} -p libgcc_s.* | tar -xf - -C $treedir/${base_libdir} 76 fi 77 install -m 0755 ${LOCALETREESRC}${bindir}/localedef $treedir/${base_bindir} 78} 79 80do_collect_bins_from_locale_tree() { 81 treedir=${WORKDIR}/locale-tree 82 83 parent=$(dirname ${localedir}) 84 mkdir -p ${PKGD}/$parent 85 tar -cf - -C $treedir/$parent -p $(basename ${localedir}) | tar -xf - -C ${PKGD}$parent 86 87 # Finalize tree by chaning all duplicate files into hard links 88 cross-localedef-hardlink -c -v ${WORKDIR}/locale-tree 89} 90 91inherit qemu 92 93python package_do_split_gconvs () { 94 import re 95 if (d.getVar('PACKAGE_NO_GCONV') == '1'): 96 bb.note("package requested not splitting gconvs") 97 return 98 99 if not d.getVar('PACKAGES'): 100 return 101 102 mlprefix = d.getVar("MLPREFIX") or "" 103 104 bpn = d.getVar('BPN') 105 libdir = d.getVar('libdir') 106 if not libdir: 107 bb.error("libdir not defined") 108 return 109 datadir = d.getVar('datadir') 110 if not datadir: 111 bb.error("datadir not defined") 112 return 113 114 gconv_libdir = oe.path.join(libdir, "gconv") 115 charmap_dir = oe.path.join(datadir, "i18n", "charmaps") 116 locales_dir = oe.path.join(datadir, "i18n", "locales") 117 binary_locales_dir = d.getVar('localedir') 118 119 def calc_gconv_deps(fn, pkg, file_regex, output_pattern, group): 120 deps = [] 121 f = open(fn, "rb") 122 c_re = re.compile(r'^copy "(.*)"') 123 i_re = re.compile(r'^include "(\w+)".*') 124 for l in f.readlines(): 125 l = l.decode("latin-1") 126 m = c_re.match(l) or i_re.match(l) 127 if m: 128 dp = legitimize_package_name('%s%s-gconv-%s' % (mlprefix, bpn, m.group(1))) 129 if not dp in deps: 130 deps.append(dp) 131 f.close() 132 if deps != []: 133 d.setVar('RDEPENDS:%s' % pkg, " ".join(deps)) 134 if bpn != 'glibc': 135 d.setVar('RPROVIDES:%s' % pkg, pkg.replace(bpn, 'glibc')) 136 137 do_split_packages(d, gconv_libdir, file_regex=r'^(.*)\.so$', output_pattern=bpn+'-gconv-%s', \ 138 description='gconv module for character set %s', hook=calc_gconv_deps, \ 139 extra_depends=bpn+'-gconv') 140 141 def calc_charmap_deps(fn, pkg, file_regex, output_pattern, group): 142 deps = [] 143 f = open(fn, "rb") 144 c_re = re.compile(r'^copy "(.*)"') 145 i_re = re.compile(r'^include "(\w+)".*') 146 for l in f.readlines(): 147 l = l.decode("latin-1") 148 m = c_re.match(l) or i_re.match(l) 149 if m: 150 dp = legitimize_package_name('%s%s-charmap-%s' % (mlprefix, bpn, m.group(1))) 151 if not dp in deps: 152 deps.append(dp) 153 f.close() 154 if deps != []: 155 d.setVar('RDEPENDS:%s' % pkg, " ".join(deps)) 156 if bpn != 'glibc': 157 d.setVar('RPROVIDES:%s' % pkg, pkg.replace(bpn, 'glibc')) 158 159 do_split_packages(d, charmap_dir, file_regex=r'^(.*)\.gz$', output_pattern=bpn+'-charmap-%s', \ 160 description='character map for %s encoding', hook=calc_charmap_deps, extra_depends='') 161 162 def calc_locale_deps(fn, pkg, file_regex, output_pattern, group): 163 deps = [] 164 f = open(fn, "rb") 165 c_re = re.compile(r'^copy "(.*)"') 166 i_re = re.compile(r'^include "(\w+)".*') 167 for l in f.readlines(): 168 l = l.decode("latin-1") 169 m = c_re.match(l) or i_re.match(l) 170 if m: 171 dp = legitimize_package_name(mlprefix+bpn+'-localedata-%s' % m.group(1)) 172 if not dp in deps: 173 deps.append(dp) 174 f.close() 175 if deps != []: 176 d.setVar('RDEPENDS:%s' % pkg, " ".join(deps)) 177 if bpn != 'glibc': 178 d.setVar('RPROVIDES:%s' % pkg, pkg.replace(bpn, 'glibc')) 179 180 do_split_packages(d, locales_dir, file_regex=r'(.*)', output_pattern=bpn+'-localedata-%s', \ 181 description='locale definition for %s', hook=calc_locale_deps, extra_depends='') 182 d.setVar('PACKAGES', d.getVar('PACKAGES', False) + ' ' + d.getVar('MLPREFIX', False) + bpn + '-gconv') 183 184 use_bin = d.getVar("GLIBC_INTERNAL_USE_BINARY_LOCALE") 185 186 dot_re = re.compile(r"(.*)\.(.*)") 187 188 # Read in supported locales and associated encodings 189 supported = {} 190 with open(oe.path.join(d.getVar('WORKDIR'), "SUPPORTED")) as f: 191 for line in f.readlines(): 192 try: 193 locale, charset = line.rstrip().split() 194 except ValueError: 195 continue 196 supported[locale] = charset 197 198 # GLIBC_GENERATE_LOCALES var specifies which locales to be generated. empty or "all" means all locales 199 to_generate = d.getVar('GLIBC_GENERATE_LOCALES') 200 if not to_generate or to_generate == 'all': 201 to_generate = sorted(supported.keys()) 202 else: 203 to_generate = to_generate.split() 204 for locale in to_generate: 205 if locale not in supported: 206 if '.' in locale: 207 charset = locale.split('.')[1] 208 else: 209 charset = 'UTF-8' 210 bb.warn("Unsupported locale '%s', assuming encoding '%s'" % (locale, charset)) 211 supported[locale] = charset 212 213 def output_locale_source(name, pkgname, locale, encoding): 214 d.setVar('RDEPENDS:%s' % pkgname, '%slocaledef %s-localedata-%s %s-charmap-%s' % \ 215 (mlprefix, mlprefix+bpn, legitimize_package_name(locale), mlprefix+bpn, legitimize_package_name(encoding))) 216 d.setVar('pkg_postinst_ontarget:%s' % pkgname, d.getVar('locale_base_postinst_ontarget') \ 217 % (locale, encoding, locale)) 218 d.setVar('pkg_postrm:%s' % pkgname, d.getVar('locale_base_postrm') % \ 219 (locale, encoding, locale)) 220 221 def output_locale_binary_rdepends(name, pkgname, locale, encoding): 222 dep = legitimize_package_name('%s-binary-localedata-%s' % (bpn, name)) 223 lcsplit = d.getVar('GLIBC_SPLIT_LC_PACKAGES') 224 if lcsplit and int(lcsplit): 225 d.appendVar('PACKAGES', ' ' + dep) 226 d.setVar('ALLOW_EMPTY:%s' % dep, '1') 227 d.setVar('RDEPENDS:%s' % pkgname, mlprefix + dep) 228 229 commands = {} 230 231 def output_locale_binary(name, pkgname, locale, encoding): 232 treedir = oe.path.join(d.getVar("WORKDIR"), "locale-tree") 233 ldlibdir = oe.path.join(treedir, d.getVar("base_libdir")) 234 path = d.getVar("PATH") 235 i18npath = oe.path.join(treedir, datadir, "i18n") 236 gconvpath = oe.path.join(treedir, "iconvdata") 237 outputpath = oe.path.join(treedir, binary_locales_dir) 238 239 use_cross_localedef = d.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF") or "0" 240 if use_cross_localedef == "1": 241 target_arch = d.getVar('TARGET_ARCH') 242 locale_arch_options = { \ 243 "arc": " --uint32-align=4 --little-endian ", \ 244 "arceb": " --uint32-align=4 --big-endian ", \ 245 "arm": " --uint32-align=4 --little-endian ", \ 246 "armeb": " --uint32-align=4 --big-endian ", \ 247 "aarch64": " --uint32-align=4 --little-endian ", \ 248 "aarch64_be": " --uint32-align=4 --big-endian ", \ 249 "sh4": " --uint32-align=4 --big-endian ", \ 250 "powerpc": " --uint32-align=4 --big-endian ", \ 251 "powerpc64": " --uint32-align=4 --big-endian ", \ 252 "powerpc64le": " --uint32-align=4 --little-endian ", \ 253 "mips": " --uint32-align=4 --big-endian ", \ 254 "mipsisa32r6": " --uint32-align=4 --big-endian ", \ 255 "mips64": " --uint32-align=4 --big-endian ", \ 256 "mipsisa64r6": " --uint32-align=4 --big-endian ", \ 257 "mipsel": " --uint32-align=4 --little-endian ", \ 258 "mipsisa32r6el": " --uint32-align=4 --little-endian ", \ 259 "mips64el":" --uint32-align=4 --little-endian ", \ 260 "mipsisa64r6el":" --uint32-align=4 --little-endian ", \ 261 "riscv64": " --uint32-align=4 --little-endian ", \ 262 "riscv32": " --uint32-align=4 --little-endian ", \ 263 "i586": " --uint32-align=4 --little-endian ", \ 264 "i686": " --uint32-align=4 --little-endian ", \ 265 "x86_64": " --uint32-align=4 --little-endian " } 266 267 if target_arch in locale_arch_options: 268 localedef_opts = locale_arch_options[target_arch] 269 else: 270 bb.error("locale_arch_options not found for target_arch=" + target_arch) 271 bb.fatal("unknown arch:" + target_arch + " for locale_arch_options") 272 273 localedef_opts += " --force --no-hard-links --no-archive --prefix=%s \ 274 --inputfile=%s/%s/i18n/locales/%s --charmap=%s %s/%s" \ 275 % (treedir, treedir, datadir, locale, encoding, outputpath, name) 276 277 cmd = "PATH=\"%s\" I18NPATH=\"%s\" GCONV_PATH=\"%s\" cross-localedef %s" % \ 278 (path, i18npath, gconvpath, localedef_opts) 279 else: # earlier slower qemu way 280 qemu = qemu_target_binary(d) 281 localedef_opts = "--force --no-hard-links --no-archive --prefix=%s \ 282 --inputfile=%s/i18n/locales/%s --charmap=%s %s" \ 283 % (treedir, datadir, locale, encoding, name) 284 285 qemu_options = d.getVar('QEMU_OPTIONS') 286 287 cmd = "PSEUDO_RELOADED=YES PATH=\"%s\" I18NPATH=\"%s\" %s -L %s \ 288 -E LD_LIBRARY_PATH=%s %s %s${base_bindir}/localedef %s" % \ 289 (path, i18npath, qemu, treedir, ldlibdir, qemu_options, treedir, localedef_opts) 290 291 commands["%s/%s" % (outputpath, name)] = cmd 292 293 bb.note("generating locale %s (%s)" % (locale, encoding)) 294 295 def output_locale(name, locale, encoding): 296 pkgname = d.getVar('MLPREFIX', False) + 'locale-base-' + legitimize_package_name(name) 297 d.setVar('ALLOW_EMPTY:%s' % pkgname, '1') 298 d.setVar('PACKAGES', '%s %s' % (pkgname, d.getVar('PACKAGES'))) 299 rprovides = ' %svirtual-locale-%s' % (mlprefix, legitimize_package_name(name)) 300 m = re.match(r"(.*)_(.*)", name) 301 if m: 302 rprovides += ' %svirtual-locale-%s' % (mlprefix, m.group(1)) 303 d.setVar('RPROVIDES:%s' % pkgname, rprovides) 304 305 if use_bin == "compile": 306 output_locale_binary_rdepends(name, pkgname, locale, encoding) 307 output_locale_binary(name, pkgname, locale, encoding) 308 elif use_bin == "precompiled": 309 output_locale_binary_rdepends(name, pkgname, locale, encoding) 310 else: 311 output_locale_source(name, pkgname, locale, encoding) 312 313 if use_bin == "compile": 314 bb.note("preparing tree for binary locale generation") 315 bb.build.exec_func("do_prep_locale_tree", d) 316 317 utf8_only = int(d.getVar('LOCALE_UTF8_ONLY') or 0) 318 utf8_is_default = int(d.getVar('LOCALE_UTF8_IS_DEFAULT') or 0) 319 320 encodings = {} 321 for locale in to_generate: 322 charset = supported[locale] 323 if utf8_only and charset != 'UTF-8': 324 continue 325 326 m = dot_re.match(locale) 327 if m: 328 base = m.group(1) 329 else: 330 base = locale 331 332 # Non-precompiled locales may be renamed so that the default 333 # (non-suffixed) encoding is always UTF-8, i.e., instead of en_US and 334 # en_US.UTF-8, we have en_US and en_US.ISO-8859-1. This implicitly 335 # contradicts SUPPORTED. 336 if use_bin == "precompiled" or not utf8_is_default: 337 output_locale(locale, base, charset) 338 else: 339 if charset == 'UTF-8': 340 output_locale(base, base, charset) 341 else: 342 output_locale('%s.%s' % (base, charset), base, charset) 343 344 def metapkg_hook(file, pkg, pattern, format, basename): 345 name = basename.split('/', 1)[0] 346 metapkg = legitimize_package_name('%s-binary-localedata-%s' % (mlprefix+bpn, name)) 347 d.appendVar('RDEPENDS:%s' % metapkg, ' ' + pkg) 348 349 if use_bin == "compile": 350 makefile = oe.path.join(d.getVar("WORKDIR"), "locale-tree", "Makefile") 351 with open(makefile, "w") as m: 352 m.write("all: %s\n\n" % " ".join(commands.keys())) 353 total = len(commands) 354 for i, (maketarget, makerecipe) in enumerate(commands.items()): 355 m.write(maketarget + ":\n") 356 m.write("\t@echo 'Progress %d/%d'\n" % (i, total)) 357 m.write("\t" + makerecipe + "\n\n") 358 d.setVar("EXTRA_OEMAKE", "-C %s ${PARALLEL_MAKE}" % (os.path.dirname(makefile))) 359 d.setVarFlag("oe_runmake", "progress", r"outof:Progress\s(\d+)/(\d+)") 360 bb.note("Executing binary locale generation makefile") 361 bb.build.exec_func("oe_runmake", d) 362 bb.note("collecting binary locales from locale tree") 363 bb.build.exec_func("do_collect_bins_from_locale_tree", d) 364 365 if use_bin in ('compile', 'precompiled'): 366 lcsplit = d.getVar('GLIBC_SPLIT_LC_PACKAGES') 367 if lcsplit and int(lcsplit): 368 do_split_packages(d, binary_locales_dir, file_regex=r'^(.*/LC_\w+)', \ 369 output_pattern=bpn+'-binary-localedata-%s', \ 370 description='binary locale definition for %s', recursive=True, 371 hook=metapkg_hook, extra_depends='', allow_dirs=True, match_path=True) 372 else: 373 do_split_packages(d, binary_locales_dir, file_regex=r'(.*)', \ 374 output_pattern=bpn+'-binary-localedata-%s', \ 375 description='binary locale definition for %s', extra_depends='', allow_dirs=True) 376 else: 377 bb.note("generation of binary locales disabled. this may break i18n!") 378 379} 380 381# We want to do this indirection so that we can safely 'return' 382# from the called function even though we're prepending 383python populate_packages:prepend () { 384 bb.build.exec_func('package_do_split_gconvs', d) 385} 386