remux.py 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271
  1. #!/usr/bin/env python3
  2. import sys, re, os, subprocess, pathlib, xattr, langcodes, logging
  3. import probe
  4. # Static config
  5. debug = False
  6. propedit_exec = 'mkvpropedit'
  7. mux_xattr = 'zib.mux'
  8. mux_vers = 1
  9. lang_xattr = 'zib.lang'
  10. lang_vers = 1
  11. lang_default = 'eng'
  12. lang_codecs = [ 'subtitle', 'audio' ]
  13. min_size = 0.9
  14. video_ext = [ r'mkv', r'mp4', r'm4v' ]
  15. subs_ext = [ r'srt' ]
  16. subs_sdh = [ r'sdh' ]
  17. subs_lang = [ r'\w\w' ]
  18. # ENV config
  19. if 'DEBUG' in os.environ:
  20. debug = bool(os.environ['DEBUG'])
  21. if 'PROPEDIT' in os.environ:
  22. propedit_exec = Pathlib.path(os.environ['PROPEDIT'])
  23. # CLI config
  24. if len(sys.argv) < 2:
  25. print(f"Usage: {sys.argv[0]} base_directory", file=sys.stderr)
  26. sys.exit(-1)
  27. root = pathlib.Path(sys.argv[1])
  28. # Logging
  29. logger = logging.getLogger(__name__)
  30. if debug:
  31. logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
  32. else:
  33. logging.basicConfig(level=logging.WARN, stream=sys.stderr)
  34. # Build filename-metadata parsing regex
  35. re_media = re.compile(
  36. r'^(.*?)' +
  37. r'(?:\.(' + '|'.join(subs_lang) + r'))?' +
  38. r'(?:\.(' + '|'.join(subs_sdh) + r'))?' +
  39. r'\.(' + '|'.join(video_ext + subs_ext) + r')$' ,
  40. re.IGNORECASE
  41. )
  42. def read_xattr(path, attr):
  43. value = None
  44. try:
  45. if attr in xattr.listxattr(path):
  46. value = int.from_bytes(xattr.getxattr(path, attr))
  47. logger.debug(f"got xattr: {attr} => {value}")
  48. except BaseException as e:
  49. logger.error(f"xattr error ({attr}): {e}")
  50. return value
  51. def write_xattr(path, attr, value):
  52. error = True
  53. try:
  54. xattr.setxattr(path, attr, value.to_bytes())
  55. error = False
  56. logger.debug(f"set xattr: {attr} => {value}")
  57. except BaseException as e:
  58. logger.error(f"xattr error ({attr}): {e}")
  59. return not error
  60. def scan(files):
  61. logger.debug(f"Scanning {len(files)} files")
  62. media = []
  63. # Scan this directory for media
  64. for file in files:
  65. info = { 'name': file, 'path': parent.joinpath(file), 'lang': None, 'sdh': False }
  66. # Parse media paths
  67. try:
  68. m = re_media.search(info['name'])
  69. if m and len(m.groups()) == 4:
  70. info['base'] = m.group(1)
  71. info['g_base'] = info['base'].lower()
  72. try:
  73. lang = langcodes.Language.get(m.group(2).lower())
  74. info['lang'] = lang.to_alpha3()
  75. except:
  76. if m.group(2):
  77. logger.warning(f"Unable to match language code: {m.group(2)}")
  78. if m.group(3):
  79. info['sdh'] = m.group(3).lower() in subs_sdh
  80. info['ext'] = m.group(4).lower()
  81. except BaseException as e:
  82. logger.error(f"Skipping file due to name error ({info['name']}): {e}")
  83. continue
  84. # Skip non-media files
  85. if 'ext' not in info:
  86. logger.debug(f"Skipping non-media file: {info['name']}")
  87. continue
  88. # Read xattrs
  89. info['mux_vers'] = read_xattr(info['path'], mux_xattr)
  90. info['lang_vers'] = read_xattr(info['path'], lang_xattr)
  91. # Loop end
  92. media.append(info)
  93. logger.debug(f"Found media: {info}")
  94. # Return
  95. return media
  96. def mux(media):
  97. logger.debug(f"Muxing {len(media)} files")
  98. # Find groups
  99. groups = {}
  100. for file in media:
  101. if 'g_base' in file:
  102. g_base = file['g_base']
  103. if file['g_base'] not in groups:
  104. groups[g_base] = []
  105. if file not in groups[g_base]:
  106. groups[g_base].append(file)
  107. # Merge groups
  108. for (g_base, group) in groups.items():
  109. logger.debug(f"Merging group: {group[0]['base']}")
  110. # Skip groups that do not need muxing
  111. if len(group) == 1:
  112. if 'mux_vers' in group[0] and group[0]['mux_vers'] and group[0]['mux_vers'] >= mux_vers:
  113. logger.debug(f"Skipping single-item group with current mux_vers: {group[0]['path']}")
  114. continue
  115. # Find all sources for the group
  116. size = 0
  117. subs = []
  118. videos = []
  119. for file in group:
  120. if file['ext'] in video_ext:
  121. logger.debug(f"Found video: {file['path'].name}")
  122. videos.append(file)
  123. else:
  124. logger.debug(f"Found subtitle: {file['path'].name}")
  125. subs.append(file)
  126. # Track total size for post-mux verification
  127. size += file['path'].stat().st_size
  128. # We should find exactly one video in the group
  129. if len(videos) != 1:
  130. logger.error(f"Did not find exactly 1 video: {group[0]['base']}")
  131. continue
  132. # Build args
  133. final = group[0]['path'].parent.joinpath(f'{group[0]['base']}.mkv')
  134. temp = final.with_suffix('.mkv-remux')
  135. args = [ 'mkvmerge', '--default-language', lang_default, '-o', temp ]
  136. args.append(videos[0]['path'])
  137. # Subtitles get language and SDH metadata
  138. # Index is always 0; these files contain only one stream
  139. for sub in subs:
  140. args.extend([ '--language', f"0:{sub['lang']}" ])
  141. if sub['sdh']:
  142. args.extend([ '--hearing-impaired-flag', '0' ])
  143. args.append(sub['path'])
  144. # mkvmerge
  145. if temp.exists():
  146. logger.error(f"Skipping because mux output exists: {temp}")
  147. continue
  148. logger.debug(f"Mux command: {args}")
  149. merge = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  150. logger.debug(f"Mux result: {merge}")
  151. # Verify
  152. if merge.returncode != 0 or not temp.exists() or temp.stat().st_size < (min_size * size):
  153. logger.error(f"Mux error: {merge.stdout.decode('utf-8')}")
  154. continue
  155. # Set xattr
  156. if not write_xattr(temp, mux_xattr, mux_vers):
  157. continue
  158. videos[0]['mux_vers'] = mux_vers
  159. # Remove sources
  160. try:
  161. for source in (videos + subs):
  162. logger.debug(f"Removing: {source['path']}")
  163. source['path'].unlink()
  164. media.remove(source)
  165. except BaseException as e:
  166. logger.error(f"Unlink error: {e}")
  167. continue
  168. # Rename
  169. err = None
  170. try:
  171. logger.debug(f"Renaming: {temp.name} -> {final.name}")
  172. temp.rename(final)
  173. videos[0]['path'] = final
  174. videos[0]['name'] = final.name
  175. media.append(videos[0])
  176. except BaseException as e:
  177. err = e
  178. if not final.exists():
  179. err = 'Output file missing'
  180. elif final.stat().st_size < (min_size * size):
  181. err = 'Output file too small'
  182. if err is not None:
  183. logger.error(f"Rename error ({final}): {e}")
  184. continue
  185. def lang(media):
  186. logger.debug(f"Language processing {len(media)} files")
  187. for file in media:
  188. logger.debug(f"Lang: {file['name']}")
  189. # Skip files tha do not need lang processing
  190. if not file['ext'] in video_ext:
  191. logger.debug(f"Skipping non-video item: {file['path']}")
  192. continue
  193. if 'lang_vers' in file and file['lang_vers'] and file['lang_vers'] >= lang_vers:
  194. logger.debug(f"Skipping item with current lang_vers: {file['path']}")
  195. continue
  196. # Probe streams
  197. streams = probe.probe(file['path'])
  198. for stream in streams:
  199. if not 'index' in stream:
  200. logger.error(f"No index for stream ({file['name']}): {stream}")
  201. continue
  202. # Find missing and undefined languages for subtitle and audio streams
  203. updates = []
  204. for stream in streams:
  205. if 'codec_type' not in stream or stream['codec_type'] not in lang_codecs:
  206. logger.debug(f"Skipping missing or ignored codec_type: {file['name']}:{stream['index']}")
  207. continue
  208. if 'TAG:language' not in stream or stream['TAG:language'] in [ None, 'und' ]:
  209. logger.debug(f"Missing or undefined language in {file['name']}:{stream['codec_type']}:{stream['index']}")
  210. updates.append(int(stream['index']))
  211. # mkvpropedit
  212. if updates:
  213. args = [ propedit_exec, file['path'] ]
  214. for index in updates:
  215. # mkvpropedit starts counting at 1
  216. args.extend([ '--edit', f"track:{index + 1}", '--set', f"language={lang_default}" ])
  217. logger.debug(f"propedit command: {args}")
  218. propedit = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  219. logger.debug(f"propedit result: {propedit}")
  220. if propedit.returncode != 0:
  221. logger.error(f"propedit error ({propedit.returncode}): {propedit.stdout.decode('utf-8')}")
  222. continue
  223. # Set xattr
  224. if not write_xattr(file['path'], lang_xattr, lang_vers):
  225. continue
  226. file['lang_vers'] = lang_vers
  227. # Execute
  228. for (parent, dirs, files) in root.walk():
  229. logger.debug(f"Walking: {parent}")
  230. media = scan(files)
  231. mux(media)
  232. lang(media)