From 7991cc8aacd88df9c8305f47d38dfd8a35b01432 Mon Sep 17 00:00:00 2001 From: Colum Paget Date: Tue, 14 Mar 2017 22:28:02 +0000 Subject: [PATCH] Fix for multipart m3u8 stream downloads. Code cleanups --- common.c | 45 ++++++++++++++++ common.h | 5 +- containerfiles.c | 57 ++++++++++----------- display.c | 1 + download.c | 4 +- download.h | 8 +-- extract_text.c | 5 +- extract_text.h | 2 +- main.c | 78 +--------------------------- selectformat.c | 1 + servicetypes.c | 130 ++++++++++++++++++++++++++++------------------- servicetypes.h | 8 +-- settings.c | 1 + 13 files changed, 174 insertions(+), 171 deletions(-) diff --git a/common.c b/common.c index ddda428..47a8cf4 100755 --- a/common.c +++ b/common.c @@ -1,4 +1,5 @@ #include "common.h" +#include "download.h" char *FileTypes[]={".flv",".mp3",".mp4",".mov",".wma",".m4a",".m4v",".wmv",".webm",".avi",".3gp","m3u8",NULL}; char *ItemSelectionArg=NULL; @@ -8,10 +9,31 @@ char *Username=NULL, *Password=NULL; char *Proxy=NULL; char *ProgName=NULL, *CmdLine=NULL, *UserAgent=NULL; int STREAMTimeout=300; +STREAM *StdIn=NULL; int Flags=0; +char *BuildURL(char *RetStr, const char *Parent, const char *SubItem) +{ +char *Proto=NULL, *Host=NULL, *Port=NULL, *Path=NULL; +char *BasePath=NULL; + +ParseURL(Parent,&Proto,&Host,&Port,NULL,NULL,&Path,NULL); +if (StrValid(Port)) BasePath=FormatStr(BasePath, "%s://%s:%s/", Proto,Host,Port); +else BasePath=FormatStr(BasePath, "%s://%s/", Proto,Host); + +//if it starts with '/' then we don't append to existing path +if (*SubItem=='/') RetStr=MCopyStr(RetStr, BasePath, SubItem, NULL); +else RetStr=MCopyStr(RetStr, BasePath, Path, "/", SubItem, NULL); + +DestroyString(Proto); +DestroyString(Host); +DestroyString(Port); +DestroyString(Path); +DestroyString(BasePath); +return(RetStr); +} char *FileTypeFromURL(char *URL) { @@ -84,3 +106,26 @@ char *Token=NULL; DestroyString(Token); } + + +int CheckForKeyboardInput() +{ +char *Tempstr=NULL; +int result=FALSE; + +if (STREAMCheckForBytes(StdIn)) +{ + Tempstr=STREAMReadLine(Tempstr,StdIn); + StripTrailingWhitespace(Tempstr); + if (StrLen(Tempstr)) + { + ListAddItem(DownloadQueue,CopyStr(NULL,Tempstr)); + if (! (Flags & FLAG_QUIET)) fprintf(stderr,"\r\nQUEUED: %s\n",Tempstr); + result=TRUE; + } +} +DestroyString(Tempstr); + +return(result); +} + diff --git a/common.h b/common.h index b6f6f86..6dbbfb2 100755 --- a/common.h +++ b/common.h @@ -2,7 +2,7 @@ #ifndef MOVGRAB_COMMON #define MOVGRAB_COMMON //This is doable through autoconf, but I'm sick of fighting with it -#define Version "3.0.0" +#define Version "3.0.1" #include "libUseful-2.6/libUseful.h" #include @@ -40,10 +40,13 @@ extern char *Username, *Password; extern char *Proxy; extern char *ProgName, *CmdLine, *UserAgent; extern int STREAMTimeout; +extern STREAM *StdIn; char *FileTypeFromURL(char *URL); +char *BuildURL(char *RetStr, const char *Parent, const char *SubItem); char *ItemCodeFromFileExtension(char *RetBuf, const char *Default, const char *URL); void VarsAddDownloadItem(const char *ItemName, const char *URL, ListNode *Vars, int AddFlags); +int CheckForKeyboardInput(); #endif diff --git a/containerfiles.c b/containerfiles.c index f747357..e2b5df0 100755 --- a/containerfiles.c +++ b/containerfiles.c @@ -119,53 +119,48 @@ DestroyString(Bandwidth); int M3UStreamDownload(STREAM *ManifestCon, const char *URL, const char *Title) { STREAM *Con=NULL; -char *Tempstr=NULL, *BasePath=NULL, *Line=NULL, *Extn=NULL; +char *Tempstr=NULL, *Line=NULL, *Extn=NULL; const char *ptr; ListNode *Segments, *Curr; int RetVal=FALSE; double BytesRead=0; Segments=ListCreate(); -ptr=strrchr(URL, '/'); -if (ptr) +Line=STREAMReadLine(Line,ManifestCon); +while (Line) { - BasePath=CopyStrLen(BasePath, URL, ptr - URL); - Line=STREAMReadLine(Line,ManifestCon); - while (Line) - { - StripLeadingWhitespace(Line); - StripTrailingWhitespace(Line); + StripLeadingWhitespace(Line); + StripTrailingWhitespace(Line); - if (*Line != '#') - { - Tempstr=MCopyStr(Tempstr, BasePath, "/", Line, NULL); - ListAddItem(Segments, CopyStr(NULL, Tempstr)); - ptr=strrchr(Line,'.'); - if (ptr) Extn=CopyStr(Extn, ptr); - } - Line=STREAMReadLine(Line,ManifestCon); + if (*Line != '#') + { + ListAddItem(Segments, BuildURL(NULL, URL, Line)); + ptr=strrchr(Line,'.'); + if (ptr) Extn=CopyStr(Extn, ptr); } +Line=STREAMReadLine(Line,ManifestCon); +} - OpenOutputFiles(Title, URL, &BytesRead); - Tempstr=SetStrLen(Tempstr,BUFSIZ); - Curr=ListGetNext(Segments); - while (Curr) +OpenOutputFiles(Title, URL, &BytesRead); +Tempstr=SetStrLen(Tempstr,BUFSIZ); +Curr=ListGetNext(Segments); +while (Curr) +{ + Con=ConnectAndRetryUntilDownload(Curr->Item, 0, 0); + if (Con) { - Con=ConnectAndRetryUntilDownload(Curr->Item, 0, 0); - if (Con) - { - RetVal=TRUE; - TransferItem(Con, Title, URL, "m3u8-stream", 0, 0, &BytesRead, FALSE); - STREAMClose(Con); - } - Curr=ListGetNext(Curr); + RetVal=TRUE; + if (Flags & FLAG_DEBUG) printf("M3U8 Segment: %s\n",Curr->Item); + TransferItem(Con, Title, URL, "m3u8-stream", 0, 0, &BytesRead, FALSE); + STREAMClose(Con); } - CloseOutputFiles(Extn); + Curr=ListGetNext(Curr); } +CloseOutputFiles(Extn); ListDestroy(Segments, DestroyString); + DestroyString(Tempstr); -DestroyString(BasePath); DestroyString(Line); DestroyString(Extn); diff --git a/display.c b/display.c index e9bd605..2e7e407 100755 --- a/display.c +++ b/display.c @@ -1,4 +1,5 @@ #include "display.h" +#include "outputfiles.h" pid_t PlayerPid=0; char *Player=NULL; diff --git a/download.c b/download.c index ece7ca5..7ab86a9 100755 --- a/download.c +++ b/download.c @@ -1,13 +1,15 @@ #include "download.h" #include "outputfiles.h" #include "display.h" +#include "containerfiles.h" +#include "servicetypes.h" /* Functions relating to connecting to hosts and downloading webpages. All the HTTP stuff is in here */ - +ListNode *DownloadQueue=NULL; extern int STREAMTimeout; STREAM *ConnectAndSendHeaders(const char *URL, int Flags, double BytesRange) diff --git a/download.h b/download.h index f16ffef..091fb1e 100755 --- a/download.h +++ b/download.h @@ -3,14 +3,16 @@ #include "common.h" +extern char *Proxy; +extern int PlayerLaunchPercent; +extern ListNode *DownloadQueue; + int DownloadItem(const char *URL, const char *Format, const char *Path, int Flags); int DownloadPage(const char *Path, int Type, const char *Title, int Flags); int TransferItem(STREAM *Con, const char *Title, const char *URL, const char *Format, double SegmentSize, double DocSize, double *BytesRead, int PrintName); STREAM *ConnectAndRetryUntilDownload(const char *URL, int Flags, double BytesRead); - -extern char *Proxy; -extern int PlayerLaunchPercent; +STREAM *ConnectAndSendHeaders(const char *URL, int Flags, double BytesRange); #endif diff --git a/extract_text.c b/extract_text.c index 6e37462..0926510 100755 --- a/extract_text.c +++ b/extract_text.c @@ -4,9 +4,10 @@ //This function Extracts Text from a line that's found between two specified //chunks of text 'ItemStart' and 'ItemEnd' -char *GenericExtractFromLine(char *Line, const char *ItemName, const char *ItemStart, const char *ItemEnd, ListNode *Vars, int ExtractFlags) +const char *GenericExtractFromLine(const char *Line, const char *ItemName, const char *ItemStart, const char *ItemEnd, ListNode *Vars, int ExtractFlags) { -char *ptr, *ptr2, *Token=NULL, *Item=NULL; +const char *ptr; +char *Token=NULL, *Item=NULL, *ptr2; int len; int GTF=0; diff --git a/extract_text.h b/extract_text.h index 02364a5..d4b5556 100755 --- a/extract_text.h +++ b/extract_text.h @@ -18,7 +18,7 @@ //This function Extracts Text from a line that's found between two specified //chunks of text 'ItemStart' and 'ItemEnd' -char *GenericExtractFromLine(char *Line, const char *ItemName, const char *ItemStart, const char *ItemEnd, ListNode *Vars, int Flags); +const char *GenericExtractFromLine(const char *Line, const char *ItemName, const char *ItemStart, const char *ItemEnd, ListNode *Vars, int Flags); void GenericTitleExtract(const char *Line, ListNode *Vars); #endif diff --git a/main.c b/main.c index 8ad91ec..57e067d 100755 --- a/main.c +++ b/main.c @@ -26,88 +26,12 @@ #include "youtube.h" #include "ehow.h" #include "display.h" +#include "settings.h" -ListNode *DownloadQueue=NULL; -STREAM *StdIn=NULL; int Type=TYPE_NONE; -int CheckForKeyboardInput() -{ -char *Tempstr=NULL; -int result=FALSE; - -if (STREAMCheckForBytes(StdIn)) -{ - Tempstr=STREAMReadLine(Tempstr,StdIn); - StripTrailingWhitespace(Tempstr); - if (StrLen(Tempstr)) - { - ListAddItem(DownloadQueue,CopyStr(NULL,Tempstr)); - if (! (Flags & FLAG_QUIET)) fprintf(stderr,"\r\nQUEUED: %s\n",Tempstr); - result=TRUE; - } -} -DestroyString(Tempstr); - -return(result); -} - - - - - -//-------- Go through the processes involved in getting a video file -int GrabMovie(char *Path, int MovType) -{ -int i; -char *Proto=NULL, *Server=NULL, *Doc=NULL, *Tempstr=NULL, *Title=NULL; -char *NextPath=NULL; -char *ptr, *Token=NULL; -int Port; -int RetVal=FALSE; - -if (!StrLen(Path)) return(FALSE); - -Type=MovType; -NextPath=CopyStr(NextPath,Path); -ParseURL(Path,&Proto,&Server,&Tempstr,NULL,NULL,&Doc,NULL); -if (Tempstr) Port=atoi(Tempstr); - -if (Proto && (strcasecmp(Proto,"https")==0) ) -{ - if (! SSLAvailable) - { - printf("SSL NOT COMPILED IN! Switching from 'https' to 'http'\n"); - NextPath=MCopyStr(NextPath,"http://",Server,"/",ptr); - } -} - -if (Type==TYPE_NONE) Type=IdentifyServiceType(Path); - - -if (Type==TYPE_NONE) -{ -if (! (Flags & FLAG_QUIET)) fprintf(stderr,"Unrecognized url type. Falling Back to 'generic youtube frontend'.\n Try using the -t option to force the service type ( \"movgrab -?\" for more details )\n"); -Type=TYPE_GENERIC; -} - -NextPath=SiteSpecificPreprocessing(NextPath, Path, Proto, Server, Port, Doc, &Type, &Title, &Flags); -if (DownloadPage(NextPath, Type, Title, Flags)) RetVal=TRUE; - -DestroyString(Tempstr); -DestroyString(Server); -DestroyString(Doc); -DestroyString(NextPath); -DestroyString(Token); -DestroyString(Title); -DestroyString(Proto); - -return(RetVal); -} - - diff --git a/selectformat.c b/selectformat.c index ccad474..da0c197 100644 --- a/selectformat.c +++ b/selectformat.c @@ -1,5 +1,6 @@ #include "selectformat.h" #include "servicetypes.h" +#include "display.h" char *GatherMatchingFormats(char *Buffer, char *Type, ListNode *Vars) { diff --git a/servicetypes.c b/servicetypes.c index 85df013..8e26ab7 100755 --- a/servicetypes.c +++ b/servicetypes.c @@ -1,6 +1,9 @@ #include "servicetypes.h" #include "containerfiles.h" #include "selectformat.h" +#include "download.h" +#include "ign.h" +#include "youtube.h" /* This file and it's header (servicetypes.h) holds all the functions and data @@ -283,34 +286,11 @@ return(Type); -char *ExtractMetacafeMediaURL(char *RetStr, char *Data, char *Start, char *End) -{ -char *Tempstr=NULL, *Token=NULL, *ptr; - - -ptr=strstr(Data,Start); -ptr+=StrLen(Start); -ptr=GetToken(ptr,End,&Token,0); -Tempstr=HTTPUnQuote(Tempstr,Token); -RetStr=MCopyStr(RetStr,Tempstr,"?__gda__=",NULL); -ptr=GetToken(Data,"gdaKey=",&Token,0); -ptr=GetToken(ptr,"&",&Token,0); -RetStr=CatStr(RetStr,Token); - -DestroyString(Tempstr); -DestroyString(Token); - - -return(RetStr); -} - - - //This function is called before we even pull the first page from a site //it is a good place to do any site-specific stuff like rewriting the //site URL to a form that's better for movgrab -char *SiteSpecificPreprocessing(char *RetBuff, char *Path, char *Proto, char *Server, int Port, char *Doc, int *Type, char **Title, int *Flags) +char *SiteSpecificPreprocessing(char *RetBuff, const char *Path, const char *Proto, const char *Server, int Port, const char *Doc, int *Type, char **Title, int *Flags) { char *Tempstr=NULL; char *NextPath=NULL; @@ -447,7 +427,7 @@ return(NextPath); //For websites with multiple videos on a page (not multiple formats of the //same video, but actual different videos) decide which one to get -void HandleMultipleMedia(int Type,char *Server,int Flags,ListNode *Vars,int MediaCount) +void HandleMultipleMedia(int Type, const char *Server, int Flags, ListNode *Vars, int MediaCount) { char *Tempstr=NULL, *ptr; int i, startpos, endpos; @@ -516,7 +496,7 @@ DestroyString(Tempstr); //Decides what to do next (Download another page, download the actual //media item, give up, etc. -int GetNextURL(int Type, char *Server, int Flags, ListNode *Vars) +int GetNextURL(int Type, const char *Server, int Flags, ListNode *Vars) { char *Tempstr=NULL, *Title=NULL, *Fmt=NULL, *ptr; int RetVal=FALSE; @@ -717,11 +697,12 @@ DestroyString(Res); // information that it can use to get a video *******************************************************************************************************/ -int ExtractItemInfo(STREAM *S, int Type, char *URL, char *Title, int Flags) +int ExtractItemInfo(STREAM *S, int Type, const char *URL, const char *Title, int Flags) { char *Tempstr=NULL, *Token=NULL, *VarName=NULL, *Server=NULL; ListNode *Vars=NULL; -char *ptr, *ptr2; +const char *ptr; +char *ptr2; int MediaCount=0, i, Port; int RetVal=FALSE, State=0; @@ -790,13 +771,8 @@ else if (strncmp(Tempstr,"#EXT-X-STREAM-INF:",18)==0) else if (strncasecmp(Tempstr,"https:",6)==0) SetVar(Vars, VarName, Tempstr); else { - ptr=strrchr(URL,'/'); - if (ptr) - { - Token=CopyStrLen(Token,URL,ptr-URL); - Token=MCatStr(Token,"/",Tempstr,NULL); - SetVar(Vars, VarName, Token); - } + Token=BuildURL(Token, URL, Tempstr); + SetVar(Vars, VarName, Token); } } else if (*Tempstr !=0) VarsAddDownloadItem("item:mp3", Tempstr, Vars, EXTRACT_GUESSTYPE); @@ -946,22 +922,22 @@ break; case TYPE_VIMEO_STAGE2: -#define VIMEO1_BASE "