remux.py 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. #!/usr/bin/env python3
  2. import sys, re, os, subprocess, pathlib, xattr, langcodes, logging
  3. import probe
  4. # Static config
  5. debug = False
  6. propedit_exec = 'mkvpropedit'
  7. mux_xattr = 'zib.mux'
  8. mux_vers = 1
  9. lang_xattr = 'zib.lang'
  10. lang_vers = 1
  11. lang_default = 'eng'
  12. lang_codecs = [ 'subtitle', 'audio' ]
  13. min_size = 0.9
  14. video_ext = [ r'mkv', r'mp4', r'm4v' ]
  15. subs_ext = [ r'srt' ]
  16. subs_sdh = [ r'sdh' ]
  17. subs_lang = [ r'\w\w' ]
  18. # Default metadata record
  19. meta_default = {
  20. 'path': None,
  21. 'name': '',
  22. 'ext': '',
  23. 'group': '',
  24. 'lang': None,
  25. 'sdh': False,
  26. 'mux_vers': None,
  27. 'lang_vers': None,
  28. }
  29. # ENV config
  30. if 'DEBUG' in os.environ:
  31. debug = bool(os.environ['DEBUG'])
  32. if 'PROPEDIT' in os.environ:
  33. propedit_exec = Pathlib.path(os.environ['PROPEDIT'])
  34. # CLI config
  35. if len(sys.argv) < 2:
  36. print(f"Usage: {sys.argv[0]} base_directory", file=sys.stderr)
  37. sys.exit(-1)
  38. root = pathlib.Path(sys.argv[1])
  39. # Logging
  40. logger = logging.getLogger(__name__)
  41. if debug:
  42. logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
  43. else:
  44. logging.basicConfig(level=logging.WARN, stream=sys.stderr)
  45. # Build filename-metadata parsing regex
  46. re_media = re.compile(
  47. r'^(.*?)' +
  48. r'(?:\.(' + '|'.join(subs_lang) + r'))?' +
  49. r'(?:\.(' + '|'.join(subs_sdh) + r'))?' +
  50. r'\.(' + '|'.join(video_ext + subs_ext) + r')$' ,
  51. re.IGNORECASE
  52. )
  53. def read_xattr(path, attr):
  54. value = None
  55. try:
  56. if attr in xattr.listxattr(path):
  57. value = int.from_bytes(xattr.getxattr(path, attr))
  58. logger.debug(f"got xattr: {attr} => {value}")
  59. except BaseException as e:
  60. logger.error(f"xattr error ({attr}): {e}")
  61. return value
  62. def write_xattr(path, attr, value):
  63. error = True
  64. try:
  65. xattr.setxattr(path, attr, value.to_bytes())
  66. error = False
  67. logger.debug(f"set xattr: {attr} => {value}")
  68. except BaseException as e:
  69. logger.error(f"xattr error ({attr}): {e}")
  70. return not error
  71. def scan(files):
  72. logger.debug(f"Scanning {len(files)} files")
  73. media = []
  74. # Scan this directory for media
  75. for file in files:
  76. info = meta_default.copy()
  77. info['path'] = parent.joinpath(file)
  78. info['name'] = file
  79. # Parse media paths
  80. try:
  81. m = re_media.search(info['name'])
  82. if m and len(m.groups()) == 4:
  83. info['group'] = m.group(1)
  84. try:
  85. lang = langcodes.Language.get(m.group(2).lower())
  86. info['lang'] = lang.to_alpha3()
  87. except:
  88. if m.group(2):
  89. logger.warning(f"Unable to match language code: {m.group(2)}")
  90. if m.group(3):
  91. info['sdh'] = m.group(3).lower() in subs_sdh
  92. info['ext'] = m.group(4).lower()
  93. except BaseException as e:
  94. logger.error(f"Skipping file due to name error ({info['name']}): {e}")
  95. continue
  96. # Skip non-media files
  97. if 'ext' not in info:
  98. logger.debug(f"Skipping non-media file: {info['name']}")
  99. continue
  100. # Read xattrs
  101. info['mux_vers'] = read_xattr(info['path'], mux_xattr)
  102. info['lang_vers'] = read_xattr(info['path'], lang_xattr)
  103. # Loop end
  104. media.append(info)
  105. logger.debug(f"Found media: {info}")
  106. # Return
  107. return media
  108. def mux(media):
  109. logger.debug(f"Muxing {len(media)} files")
  110. # Find groups
  111. groups = {}
  112. for file in media:
  113. if not file['group']:
  114. logger.debug(f"Skipping non-grouped file: {file['name']}")
  115. continue
  116. gnorm = file['group'].lower()
  117. if gnorm not in groups:
  118. groups[gnorm] = []
  119. if file not in groups[gnorm]:
  120. groups[gnorm].append(file)
  121. # Merge groups
  122. for group in groups.values():
  123. logger.debug(f"Muxing group: {group[0]['group']}")
  124. # Skip groups that do not need muxing
  125. if len(group) == 1:
  126. if 'mux_vers' in group[0] and group[0]['mux_vers'] and group[0]['mux_vers'] >= mux_vers:
  127. logger.debug(f"Skipping single-item group with current mux_vers: {group[0]['path']}")
  128. continue
  129. # Find all sources for the group
  130. size = 0
  131. subs = []
  132. videos = []
  133. for file in group:
  134. if file['ext'] in video_ext:
  135. logger.debug(f"Found video: {file['name']}")
  136. videos.append(file)
  137. elif file['ext'] in subs_ext:
  138. logger.debug(f"Found subtitle: {file['name']}")
  139. subs.append(file)
  140. else:
  141. logger.warning(f"Skipping unexpected source while muxing: {file['name']}")
  142. # Track total size for post-mux verification
  143. size += file['path'].stat().st_size
  144. # We should find exactly one video in the group
  145. if len(videos) != 1:
  146. logger.error(f"Did not find exactly 1 video in group: {group[0]['group']}")
  147. continue
  148. # Build args
  149. final = videos[0]['path'].parent.joinpath(f'{videos[0]['group']}.mkv')
  150. temp = final.with_suffix('.mkv-remux')
  151. args = [ 'mkvmerge', '--default-language', lang_default, '-o', temp ]
  152. args.append(videos[0]['path'])
  153. # Subtitles get language and SDH metadata
  154. # Index is always 0; these files contain only one stream
  155. for sub in subs:
  156. args.extend([ '--language', f"0:{sub['lang']}" ])
  157. if sub['sdh']:
  158. args.extend([ '--hearing-impaired-flag', '0' ])
  159. args.append(sub['path'])
  160. # mkvmerge
  161. if temp.exists():
  162. logger.error(f"Skipping because mux output exists: {temp}")
  163. continue
  164. logger.debug(f"Mux command: {args}")
  165. merge = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  166. logger.debug(f"Mux result: {merge}")
  167. # Verify
  168. err = None
  169. if merge.returncode != 0:
  170. err = f"Non-zero exit code: {merge.returncode}"
  171. elif not temp.exists():
  172. err = 'file missing'
  173. elif temp.stat().st_size < (min_size * size):
  174. err = 'file too small'
  175. if err is not None:
  176. logger.error(f"Mux error ({err}):\n{merge.stdout.decode('utf-8')}")
  177. continue
  178. # Set xattr
  179. if not write_xattr(temp, mux_xattr, mux_vers):
  180. continue
  181. # Remove sources
  182. try:
  183. for source in (videos + subs):
  184. logger.debug(f"Removing: {source['path']}")
  185. source['path'].unlink()
  186. media.remove(source)
  187. except BaseException as e:
  188. logger.error(f"Unlink error: {e}")
  189. continue
  190. # Rename
  191. err = None
  192. try:
  193. logger.debug(f"Renaming: {temp.name} -> {final.name}")
  194. temp.rename(final)
  195. info = meta_default.copy()
  196. info['path'] = final
  197. info['name'] = final.name
  198. info['ext'] = 'mkv'
  199. info['group'] = videos[0]['group']
  200. info['mux_vers'] = mux_vers
  201. media.append(info)
  202. except BaseException as e:
  203. err = e
  204. if not final.exists():
  205. err = 'file missing'
  206. elif final.stat().st_size < (min_size * size):
  207. err = 'file too small'
  208. if err is not None:
  209. logger.error(f"Rename error ({final}): {err}")
  210. continue
  211. def lang(media):
  212. logger.debug(f"Language processing {len(media)} files")
  213. for file in media:
  214. logger.debug(f"Lang: {file['name']}")
  215. # Skip files that do not need lang processing
  216. if not file['ext'] in video_ext:
  217. logger.debug(f"Skipping non-video item: {file['path']}")
  218. continue
  219. if 'lang_vers' in file and file['lang_vers'] and file['lang_vers'] >= lang_vers:
  220. logger.debug(f"Skipping item with current lang_vers: {file['path']}")
  221. continue
  222. # Probe streams
  223. streams = probe.probe(file['path'])
  224. for stream in streams:
  225. if not 'index' in stream:
  226. logger.error(f"No index for stream ({file['name']}): {stream}")
  227. continue
  228. # Find missing and undefined languages for subtitle and audio streams
  229. updates = []
  230. for stream in streams:
  231. if 'codec_type' not in stream or stream['codec_type'] not in lang_codecs:
  232. logger.debug(f"Skipping missing or ignored codec_type: {file['name']}:{stream['index']}")
  233. continue
  234. if 'TAG:language' not in stream or stream['TAG:language'] in [ None, 'und' ]:
  235. logger.debug(f"Missing or undefined language in {file['name']}:{stream['codec_type']}:{stream['index']}")
  236. updates.append(int(stream['index']))
  237. # mkvpropedit
  238. if updates:
  239. args = [ propedit_exec, file['path'] ]
  240. for index in updates:
  241. # mkvpropedit starts counting at 1
  242. args.extend([ '--edit', f"track:{index + 1}", '--set', f"language={lang_default}" ])
  243. logger.debug(f"propedit command: {args}")
  244. propedit = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  245. logger.debug(f"propedit result: {propedit}")
  246. if propedit.returncode != 0:
  247. logger.error(f"propedit error ({propedit.returncode}): {propedit.stdout.decode('utf-8')}")
  248. continue
  249. # Set xattr
  250. if not write_xattr(file['path'], lang_xattr, lang_vers):
  251. continue
  252. file['lang_vers'] = lang_vers
  253. # Execute
  254. logger.debug(f"Root: {root}")
  255. if not root.is_dir():
  256. print(f"Invalid root directory: {root}", file=sys.stderr)
  257. sys.exit(-1)
  258. for (parent, dirs, files) in root.walk():
  259. logger.debug(f"Walking: {parent}")
  260. media = scan(files)
  261. mux(media)
  262. lang(media)