From d82b912341398431aa06ad2072d74f68f462f59d Mon Sep 17 00:00:00 2001 From: Etienne Charland Date: Tue, 31 Jan 2017 01:36:21 -0500 Subject: [PATCH 1/3] Added support for Vorbis video, Opus audio, 60fps, as well as various HD formats that are only available by parsing the Dash Manifest. Added ProgressEventArgs.ProgressBytes. --- .../YoutubeExtractor/AudioDownloader.cs | 2 +- .../YoutubeExtractor/AudioType.cs | 1 + .../YoutubeExtractor/DownloadUrlResolver.cs | 97 ++++++++++++++++++- YoutubeExtractor/YoutubeExtractor/FlvFile.cs | 2 +- .../YoutubeExtractor/ProgressEventArgs.cs | 8 +- .../YoutubeExtractor/VideoDownloader.cs | 7 +- .../YoutubeExtractor/VideoInfo.cs | 34 ++++++- .../YoutubeExtractor/YoutubeExtractor.csproj | 6 +- .../packages.YoutubeExtractor.config | 2 +- 9 files changed, 147 insertions(+), 12 deletions(-) diff --git a/YoutubeExtractor/YoutubeExtractor/AudioDownloader.cs b/YoutubeExtractor/YoutubeExtractor/AudioDownloader.cs index d359e8d..ad444e3 100644 --- a/YoutubeExtractor/YoutubeExtractor/AudioDownloader.cs +++ b/YoutubeExtractor/YoutubeExtractor/AudioDownloader.cs @@ -102,7 +102,7 @@ private void ExtractAudio(string path) { if (this.AudioExtractionProgressChanged != null) { - this.AudioExtractionProgressChanged(this, new ProgressEventArgs(args.ProgressPercentage)); + this.AudioExtractionProgressChanged(this, args); } }; diff --git a/YoutubeExtractor/YoutubeExtractor/AudioType.cs b/YoutubeExtractor/YoutubeExtractor/AudioType.cs index e78e43d..a1c827b 100644 --- a/YoutubeExtractor/YoutubeExtractor/AudioType.cs +++ b/YoutubeExtractor/YoutubeExtractor/AudioType.cs @@ -5,6 +5,7 @@ public enum AudioType Aac, Mp3, Vorbis, + Opus, /// /// The audio type is unknown. This can occur if YoutubeExtractor is not up-to-date. diff --git a/YoutubeExtractor/YoutubeExtractor/DownloadUrlResolver.cs b/YoutubeExtractor/YoutubeExtractor/DownloadUrlResolver.cs index 6896edf..7e48470 100644 --- a/YoutubeExtractor/YoutubeExtractor/DownloadUrlResolver.cs +++ b/YoutubeExtractor/YoutubeExtractor/DownloadUrlResolver.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Net; using System.Text.RegularExpressions; +using System.Xml; using Newtonsoft.Json.Linq; namespace YoutubeExtractor @@ -91,10 +92,22 @@ public static IEnumerable GetDownloadUrls(string videoUrl, bool decry IEnumerable downloadUrls = ExtractDownloadUrls(json); - IEnumerable infos = GetVideoInfos(downloadUrls, videoTitle).ToList(); + List infos = GetVideoInfos(downloadUrls, videoTitle).ToList(); + + string dashManifestUrl = GetDashManifest(json); string htmlPlayerVersion = GetHtml5PlayerVersion(json); + // Query dash manifest URL for additional formats + if (!string.IsNullOrEmpty(dashManifestUrl)) { + string signature = ExtractSignatureFromManifest(dashManifestUrl); + if (!string.IsNullOrEmpty(signature)) { + string decrypt = GetDecipheredSignature(signature, htmlPlayerVersion); + dashManifestUrl = dashManifestUrl.Replace(signature, decrypt).Replace("/s/", "/signature/"); + } + ParseDashManifest(dashManifestUrl, infos, videoTitle); + } + foreach (VideoInfo info in infos) { info.HtmlPlayerVersion = htmlPlayerVersion; @@ -228,6 +241,20 @@ private static string GetDecipheredSignature(string htmlPlayerVersion, string si return Decipherer.DecipherWithVersion(signature, htmlPlayerVersion); } + /// + /// Extracts the signature from the DASH Manifest which is located after /s/. + /// + /// The DASH Manifest URL to extract from. + /// The extracted signature. + private static string ExtractSignatureFromManifest(string manifestUrl) { + string[] Params = manifestUrl.Split('/'); + for (int i = 0; i < Params.Length; i++) { + if (Params[i] == "s" && i < Params.Length - 1) + return Params[i + 1]; + } + return string.Empty; + } + private static string GetHtml5PlayerVersion(JObject json) { var regex = new Regex(@"player-(.+?).js"); @@ -287,6 +314,28 @@ private static IEnumerable GetVideoInfos(IEnumerable return downLoadInfos; } + public static VideoInfo GetSingleVideoInfo(int formatCode, string queryUrl, string videoTitle, bool requiresDecryption) { + var Params = HttpHelper.ParseQueryString(queryUrl); + + VideoInfo info = VideoInfo.Defaults.SingleOrDefault(videoInfo => videoInfo.FormatCode == formatCode); + + if (info != null) { + long fileSize = Params.ContainsKey("clen") ? long.Parse(Params["clen"]) : 0; + info = new VideoInfo(info) { + DownloadUrl = queryUrl, + Title = videoTitle, + RequiresDecryption = requiresDecryption, + FileSize = fileSize + }; + } else { + info = new VideoInfo(formatCode) { + DownloadUrl = queryUrl + }; + } + + return info; + } + private static string GetVideoTitle(JObject json) { JToken title = json["args"]["title"]; @@ -294,6 +343,12 @@ private static string GetVideoTitle(JObject json) return title == null ? String.Empty : title.ToString(); } + private static string GetDashManifest(JObject json) { + JToken manifest = json["args"]["dashmpd"]; + + return manifest == null ? String.Empty : manifest.ToString(); + } + private static bool IsVideoUnavailable(string pageSource) { const string unavailableContainer = "
"; @@ -317,6 +372,46 @@ private static JObject LoadJson(string url) return JObject.Parse(extractedJson); } + private static void ParseDashManifest(string dashManifestUrl, List previousFormats, string videoTitle) { + string pageSource = HttpHelper.DownloadString(dashManifestUrl); + + XmlDocument doc = new XmlDocument(); + XmlNamespaceManager docNamespace = new XmlNamespaceManager(doc.NameTable); + docNamespace.AddNamespace("urn", "urn:mpeg:DASH:schema:MPD:2011"); + docNamespace.AddNamespace("yd", "http://youtube.com/yt/2012/10/10"); + doc.LoadXml(pageSource); + XmlNodeList ManifestList = doc.SelectNodes("//urn:Representation", docNamespace); + foreach (XmlElement item in ManifestList) { + int FormatCode = int.Parse(item.GetAttribute("id")); + XmlNode BaseUrl = item.GetElementsByTagName("BaseURL").Item(0); + VideoInfo info = GetSingleVideoInfo(FormatCode, BaseUrl.InnerText, videoTitle, false); + if (item.HasAttribute("height")) + info.Resolution = int.Parse(item.GetAttribute("height")); + if (item.HasAttribute("frameRate")) + info.FrameRate = int.Parse(item.GetAttribute("frameRate")); + + VideoInfo DeleteItem = previousFormats.SingleOrDefault(v => v.FormatCode == FormatCode); + if (DeleteItem != null) + previousFormats.Remove(DeleteItem); + previousFormats.Add(info); + } + } + + /// + /// Non-DASH videos don't provide file size. Queries the server to know the stream size. + /// + /// The information of the stream to get the size for. + /// The stream size in bytes. + public static void QueryStreamSize(VideoInfo info) { + if (info.RequiresDecryption) + DecryptDownloadUrl(info); + + var request = (HttpWebRequest)WebRequest.Create(info.DownloadUrl); + using (WebResponse response = request.GetResponse()) { + info.FileSize = (int)response.ContentLength; + } + } + private static void ThrowYoutubeParseException(Exception innerException, string videoUrl) { throw new YoutubeParseException("Could not parse the Youtube page for URL " + videoUrl + "\n" + diff --git a/YoutubeExtractor/YoutubeExtractor/FlvFile.cs b/YoutubeExtractor/YoutubeExtractor/FlvFile.cs index 6b4692b..1b51457 100644 --- a/YoutubeExtractor/YoutubeExtractor/FlvFile.cs +++ b/YoutubeExtractor/YoutubeExtractor/FlvFile.cs @@ -93,7 +93,7 @@ public void ExtractStreams() if (this.ConversionProgressChanged != null) { - this.ConversionProgressChanged(this, new ProgressEventArgs(progress)); + this.ConversionProgressChanged(this, new ProgressEventArgs((int)this.fileOffset, progress)); } } diff --git a/YoutubeExtractor/YoutubeExtractor/ProgressEventArgs.cs b/YoutubeExtractor/YoutubeExtractor/ProgressEventArgs.cs index 1217c57..1c78ddf 100644 --- a/YoutubeExtractor/YoutubeExtractor/ProgressEventArgs.cs +++ b/YoutubeExtractor/YoutubeExtractor/ProgressEventArgs.cs @@ -4,8 +4,9 @@ namespace YoutubeExtractor { public class ProgressEventArgs : EventArgs { - public ProgressEventArgs(double progressPercentage) + public ProgressEventArgs(int progressBytes, double progressPercentage) { + this.ProgressBytes = progressBytes; this.ProgressPercentage = progressPercentage; } @@ -14,6 +15,11 @@ public ProgressEventArgs(double progressPercentage) ///
public bool Cancel { get; set; } + /// + /// Gets the progress in bytes downloaded. + /// + public int ProgressBytes { get; private set; } + /// /// Gets the progress percentage in a range from 0.0 to 100.0. /// diff --git a/YoutubeExtractor/YoutubeExtractor/VideoDownloader.cs b/YoutubeExtractor/YoutubeExtractor/VideoDownloader.cs index 2633278..06bbeee 100644 --- a/YoutubeExtractor/YoutubeExtractor/VideoDownloader.cs +++ b/YoutubeExtractor/YoutubeExtractor/VideoDownloader.cs @@ -25,6 +25,11 @@ public VideoDownloader(VideoInfo video, string savePath, int? bytesToDownload = /// public event EventHandler DownloadProgressChanged; + /// + /// Returns the size of the current download in bytes. + /// + public int DownloadSize { get; private set; } + /// /// Starts the video download. /// @@ -59,7 +64,7 @@ public override void Execute() copiedBytes += bytes; - var eventArgs = new ProgressEventArgs((copiedBytes * 1.0 / response.ContentLength) * 100); + var eventArgs = new ProgressEventArgs(copiedBytes, (copiedBytes * 1.0 / response.ContentLength) * 100); if (this.DownloadProgressChanged != null) { diff --git a/YoutubeExtractor/YoutubeExtractor/VideoInfo.cs b/YoutubeExtractor/YoutubeExtractor/VideoInfo.cs index e52db40..c6bd574 100644 --- a/YoutubeExtractor/YoutubeExtractor/VideoInfo.cs +++ b/YoutubeExtractor/YoutubeExtractor/VideoInfo.cs @@ -49,24 +49,39 @@ public class VideoInfo new VideoInfo(271, VideoType.WebM, 1440, false, AudioType.Unknown, 0, AdaptiveType.Video), new VideoInfo(272, VideoType.WebM, 2160, false, AudioType.Unknown, 0, AdaptiveType.Video), new VideoInfo(278, VideoType.WebM, 144, false, AudioType.Unknown, 0, AdaptiveType.Video), + new VideoInfo(302, VideoType.WebM, 720, false, AudioType.Unknown, 0, AdaptiveType.Video, 60), + new VideoInfo(303, VideoType.WebM, 1080, false, AudioType.Unknown, 0, AdaptiveType.Video, 60), + new VideoInfo(308, VideoType.WebM, 1440, false, AudioType.Unknown, 0, AdaptiveType.Video, 60), + new VideoInfo(313, VideoType.WebM, 2160, false, AudioType.Unknown, 0, AdaptiveType.Video), + new VideoInfo(315, VideoType.WebM, 2160, false, AudioType.Unknown, 0, AdaptiveType.Video, 60), + new VideoInfo(298, VideoType.Mp4, 720, false, AudioType.Unknown, 0, AdaptiveType.Video, 60), + new VideoInfo(299, VideoType.Mp4, 1080, false, AudioType.Unknown, 0, AdaptiveType.Video, 60), + new VideoInfo(266, VideoType.Mp4, 2160, false, AudioType.Unknown, 0, AdaptiveType.Video), /* Adaptive (aka DASH) - Audio */ new VideoInfo(139, VideoType.Mp4, 0, false, AudioType.Aac, 48, AdaptiveType.Audio), new VideoInfo(140, VideoType.Mp4, 0, false, AudioType.Aac, 128, AdaptiveType.Audio), new VideoInfo(141, VideoType.Mp4, 0, false, AudioType.Aac, 256, AdaptiveType.Audio), new VideoInfo(171, VideoType.WebM, 0, false, AudioType.Vorbis, 128, AdaptiveType.Audio), - new VideoInfo(172, VideoType.WebM, 0, false, AudioType.Vorbis, 192, AdaptiveType.Audio) + new VideoInfo(172, VideoType.WebM, 0, false, AudioType.Vorbis, 192, AdaptiveType.Audio), + new VideoInfo(249, VideoType.WebM, 0, false, AudioType.Opus, 50, AdaptiveType.Audio), + new VideoInfo(250, VideoType.WebM, 0, false, AudioType.Opus, 70, AdaptiveType.Audio), + new VideoInfo(251, VideoType.WebM, 0, false, AudioType.Opus, 160, AdaptiveType.Audio), }; internal VideoInfo(int formatCode) - : this(formatCode, VideoType.Unknown, 0, false, AudioType.Unknown, 0, AdaptiveType.None) + : this(formatCode, VideoType.Unknown, 0, false, AudioType.Unknown, 0, AdaptiveType.None, 0) { } internal VideoInfo(VideoInfo info) - : this(info.FormatCode, info.VideoType, info.Resolution, info.Is3D, info.AudioType, info.AudioBitrate, info.AdaptiveType) + : this(info.FormatCode, info.VideoType, info.Resolution, info.Is3D, info.AudioType, info.AudioBitrate, info.AdaptiveType, info.FrameRate) { } private VideoInfo(int formatCode, VideoType videoType, int resolution, bool is3D, AudioType audioType, int audioBitrate, AdaptiveType adaptiveType) + : this(formatCode, videoType, resolution, is3D, audioType, audioBitrate, adaptiveType, 0) + { } + + private VideoInfo(int formatCode, VideoType videoType, int resolution, bool is3D, AudioType audioType, int audioBitrate, AdaptiveType adaptiveType, int frameRate) { this.FormatCode = formatCode; this.VideoType = videoType; @@ -75,6 +90,7 @@ private VideoInfo(int formatCode, VideoType videoType, int resolution, bool is3D this.AudioType = audioType; this.AudioBitrate = audioBitrate; this.AdaptiveType = adaptiveType; + this.FrameRate = frameRate; } /// @@ -92,6 +108,11 @@ private VideoInfo(int formatCode, VideoType videoType, int resolution, bool is3D /// The approximate audio bitrate in kbit/s, or 0 if the bitrate is unknown. public int AudioBitrate { get; private set; } + /// + /// The frame rate of the video, usually 60 or 0 for unspecified. + /// + public int FrameRate { get; internal set; } + /// /// Gets the audio extension. /// @@ -153,11 +174,16 @@ public bool CanExtractAudio /// public bool RequiresDecryption { get; internal set; } + /// + /// Gets the size of the stream in bytes. + /// + public long FileSize { get; internal set; } + /// /// Gets the resolution of the video. /// /// The resolution of the video, or 0 if the resolution is unkown. - public int Resolution { get; private set; } + public int Resolution { get; internal set; } /// /// Gets the video title. diff --git a/YoutubeExtractor/YoutubeExtractor/YoutubeExtractor.csproj b/YoutubeExtractor/YoutubeExtractor/YoutubeExtractor.csproj index 9cb7519..a15ef8b 100644 --- a/YoutubeExtractor/YoutubeExtractor/YoutubeExtractor.csproj +++ b/YoutubeExtractor/YoutubeExtractor/YoutubeExtractor.csproj @@ -35,12 +35,14 @@ true - - ..\packages\Newtonsoft.Json.5.0.8\lib\net35\Newtonsoft.Json.dll + + ..\..\..\NaturalGroundingPlayer\packages\Newtonsoft.Json.9.0.1\lib\net35\Newtonsoft.Json.dll + True + diff --git a/YoutubeExtractor/YoutubeExtractor/packages.YoutubeExtractor.config b/YoutubeExtractor/YoutubeExtractor/packages.YoutubeExtractor.config index 9520f36..ff91865 100644 --- a/YoutubeExtractor/YoutubeExtractor/packages.YoutubeExtractor.config +++ b/YoutubeExtractor/YoutubeExtractor/packages.YoutubeExtractor.config @@ -1,4 +1,4 @@  - + \ No newline at end of file From ae9acbc5fb2b60e04c0a88823493c428ac8bbfb9 Mon Sep 17 00:00:00 2001 From: Etienne Charland Date: Tue, 31 Jan 2017 11:42:47 -0500 Subject: [PATCH 2/3] Fix Forbidden error --- YoutubeExtractor/YoutubeExtractor/Decipherer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/YoutubeExtractor/YoutubeExtractor/Decipherer.cs b/YoutubeExtractor/YoutubeExtractor/Decipherer.cs index 65014cb..e4a22b5 100644 --- a/YoutubeExtractor/YoutubeExtractor/Decipherer.cs +++ b/YoutubeExtractor/YoutubeExtractor/Decipherer.cs @@ -13,7 +13,7 @@ public static string DecipherWithVersion(string cipher, string cipherVersion) string js = HttpHelper.DownloadString(jsUrl); //Find "C" in this: var A = B.sig||C (B.s) - string functNamePattern = @"\.sig\s*\|\|([a-zA-Z0-9\$]+)\("; //Regex Formed To Find Word or DollarSign + string functNamePattern = @"\""signature"",\s?([a-zA-Z0-9\$]+)\("; var funcName = Regex.Match(js, functNamePattern).Groups[1].Value; From 406172e7120cdf5aade026c08e6fd11104563456 Mon Sep 17 00:00:00 2001 From: Etienne Charland Date: Tue, 31 Jan 2017 11:58:42 -0500 Subject: [PATCH 3/3] Fixed DownloadSize not being set --- YoutubeExtractor/YoutubeExtractor/VideoDownloader.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/YoutubeExtractor/YoutubeExtractor/VideoDownloader.cs b/YoutubeExtractor/YoutubeExtractor/VideoDownloader.cs index 06bbeee..cb6301a 100644 --- a/YoutubeExtractor/YoutubeExtractor/VideoDownloader.cs +++ b/YoutubeExtractor/YoutubeExtractor/VideoDownloader.cs @@ -51,6 +51,7 @@ public override void Execute() { using (Stream source = response.GetResponseStream()) { + DownloadSize = (int)response.ContentLength; using (FileStream target = File.Open(this.SavePath, FileMode.Create, FileAccess.Write)) { var buffer = new byte[1024];