libzypp  17.31.0
Downloader.cc
Go to the documentation of this file.
1 
2 #include <iostream>
3 #include <fstream>
4 
5 #include <zypp/base/LogTools.h>
6 #include <zypp/base/Gettext.h>
7 #include <zypp/base/String.h>
8 #include <zypp/base/Regex.h>
9 #include <zypp-core/OnMediaLocation>
10 #include <zypp/MediaSetAccess.h>
11 #include <zypp/Fetcher.h>
12 #include <zypp/Locale.h>
13 #include <zypp/ZConfig.h>
15 #include <zypp/repo/susetags/Downloader.h>
16 #include <zypp-core/parser/ParseException>
18 #include <zypp-core/base/UserRequestException>
19 
20 using namespace zypp::parser;
21 using namespace zypp::parser::susetags;
22 
23 namespace zypp
24 {
25 namespace repo
26 {
27 namespace susetags
28 {
29 
30 Downloader::Downloader( const RepoInfo &repoinfo, const Pathname &delta_dir )
31  : repo::Downloader(repoinfo), _delta_dir(delta_dir)
32 {
33 }
34 
36 {
37  RepoStatus ret( media.provideOptionalFile( repoInfo().path() + "/content") );
38  if ( !ret.empty() ) // else: mandatory master index is missing
39  ret = ret && RepoStatus( media.provideOptionalFile( "/media.1/media" ) );
40  // else: mandatory master index is missing -> stay empty
41  return ret;
42 }
43 
44 // search old repository file file to run the delta algorithm on
45 static Pathname search_deltafile( const Pathname &dir, const Pathname &file )
46 {
47  Pathname deltafile(dir + file.basename());
48  if (PathInfo(deltafile).isExist())
49  return deltafile;
50  return Pathname();
51 }
52 
53 
56  const Pathname &dest_dir,
57  const ProgressData::ReceiverFnc & progress )
58 {
59  downloadMediaInfo( dest_dir, media );
60 
61  Pathname masterIndex( repoInfo().path() / "/content" );
62  defaultDownloadMasterIndex( media, dest_dir, masterIndex );
63 
64  // Content file first to get the repoindex
65  {
66  Pathname inputfile( dest_dir / masterIndex );
67  ContentFileReader content;
68  content.setRepoIndexConsumer( bind( &Downloader::consumeIndex, this, _1 ) );
69  content.parse( inputfile );
70  }
71  if ( ! _repoindex )
72  {
73  ZYPP_THROW( ParseException( (dest_dir+repoInfo().path()).asString() + ": " + "No repository index in content file." ) );
74  }
75  MIL << "RepoIndex: " << _repoindex << endl;
76  if ( _repoindex->metaFileChecksums.empty() )
77  {
78  ZYPP_THROW( ParseException( (dest_dir+repoInfo().path()).asString() + ": " + "No metadata checksums in content file." ) );
79  }
80  if ( _repoindex->signingKeys.empty() )
81  {
82  WAR << "No signing keys defined." << endl;
83  }
84 
85  // Prepare parsing
86  Pathname descr_dir = _repoindex->descrdir; // path below reporoot
87  //_datadir = _repoIndex->datadir; // path below reporoot
88 
89  std::map<std::string,RepoIndex::FileChecksumMap::const_iterator> availablePackageTranslations;
90 
91  for_( it, _repoindex->metaFileChecksums.begin(), _repoindex->metaFileChecksums.end() )
92  {
93  // omit unwanted translations
94  if ( str::hasPrefix( it->first, "packages" ) )
95  {
96  static const str::regex rx_packages( "^packages((.gz)?|(.([^.]*))(.gz)?)$" );
97  str::smatch what;
98  if ( str::regex_match( it->first, what, rx_packages ) )
99  {
100  if ( what[4].empty() // packages(.gz)?
101  || what[4] == "DU"
102  || what[4] == "en" )
103  { ; /* always downloaded */ }
104  else if ( what[4] == "FL" )
105  { continue; /* never downloaded */ }
106  else
107  {
108  // remember and decide later
109  availablePackageTranslations[what[4]] = it;
110  continue;
111  }
112  }
113  else
114  continue; // discard
115  }
116  else if ( it->first == "patterns.pat"
117  || it->first == "patterns.pat.gz" )
118  {
119  // take all patterns in one go
120  }
121  else if ( str::endsWith( it->first, ".pat" )
122  || str::endsWith( it->first, ".pat.gz" ) )
123  {
124 
125  // *** see also zypp/parser/susetags/RepoParser.cc ***
126 
127  // omit unwanted patterns, see https://bugzilla.novell.com/show_bug.cgi?id=298716
128  // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
129  // split at dots, take .pat or .pat.gz into account
130 
131  std::vector<std::string> patparts;
132  unsigned archpos = 2;
133  // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
134  unsigned count = str::split( it->first, std::back_inserter(patparts), "." );
135  if ( patparts[count-1] == "gz" )
136  archpos++;
137 
138  if ( count > archpos )
139  {
140  try // might by an invalid architecture
141  {
142  Arch patarch( patparts[count-archpos] );
144  {
145  // discard, if not compatible
146  MIL << "Discarding pattern " << it->first << endl;
147  continue;
148  }
149  }
150  catch ( const Exception & excpt )
151  {
152  WAR << "Pattern file name does not contain recognizable architecture: " << it->first << endl;
153  // keep .pat file if it doesn't contain an recognizable arch
154  }
155  }
156  }
157  MIL << "adding job " << it->first << endl;
158  auto location = std::move( OnMediaLocation( repoInfo().path() + descr_dir + it->first, 1 )
159  .setChecksum( it->second )
160  .setDeltafile( search_deltafile(_delta_dir + descr_dir, it->first) ));
161  enqueueDigested( location, FileChecker() );
162  }
163 
164  // check whether to download more package translations:
165  {
166  auto fnc_checkTransaltions( [&]( const Locale & locale_r ) {
167  for ( Locale toGet( locale_r ); toGet; toGet = toGet.fallback() )
168  {
169  auto it( availablePackageTranslations.find( toGet.code() ) );
170  if ( it != availablePackageTranslations.end() )
171  {
172  auto mit( it->second );
173  MIL << "adding job " << mit->first << endl;
174  auto location = std::move( OnMediaLocation( repoInfo().path() + descr_dir + mit->first, 1 )
175  .setChecksum( mit->second )
176  .setDeltafile( search_deltafile(_delta_dir + descr_dir, mit->first) ));
177  enqueueDigested(location, FileChecker());
178  break;
179  }
180  }
181  });
182  for ( const Locale & it : ZConfig::instance().repoRefreshLocales() )
183  {
184  fnc_checkTransaltions( it );
185  }
186  fnc_checkTransaltions( ZConfig::instance().textLocale() );
187  }
188 
189  for_( it, _repoindex->mediaFileChecksums.begin(), _repoindex->mediaFileChecksums.end() )
190  {
191  // Repo adopts license files listed in HASH
192  if ( it->first != "license.tar.gz" )
193  continue;
194 
195  MIL << "adding job " << it->first << endl;
196  auto location = std::move( OnMediaLocation ( repoInfo().path() + it->first, 1 )
197  .setChecksum( it->second )
198  .setDeltafile( search_deltafile( _delta_dir, it->first ) ));
199  enqueueDigested(location, FileChecker() );
200  }
201 
202  for_( it, _repoindex->signingKeys.begin(),_repoindex->signingKeys.end() )
203  {
204  MIL << "adding job " << it->first << endl;
205  OnMediaLocation location( repoInfo().path() + it->first, 1 );
206  location.setChecksum( it->second );
207  enqueueDigested(location);
208  }
209 
210  start( dest_dir, media );
211 }
212 
213 void Downloader::consumeIndex( const RepoIndex_Ptr & data_r )
214 {
215  MIL << "Consuming repo index" << endl;
216  _repoindex = data_r;
217 }
218 
219 }// ns susetags
220 }// ns source
221 } // ns zypp
std::string asString(const Patch::Category &obj)
Definition: Patch.cc:122
#define MIL
Definition: Logger.h:96
void defaultDownloadMasterIndex(MediaSetAccess &media_r, const Pathname &destdir_r, const Pathname &masterIndex_r)
Common workflow downloading a (signed) master index file.
Definition: Downloader.cc:141
Pathname deltafile
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Describes a resource file located on a medium.
Regular expression.
Definition: Regex.h:94
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:823
Pathname provideOptionalFile(const Pathname &file, unsigned media_nr=1)
Provides an optional file from media media_nr.
RepoStatus status(MediaSetAccess &media) override
Status of the remote repository.
Definition: Downloader.cc:35
Architecture.
Definition: Arch.h:36
Locale fallback() const
Return the fallback locale for this locale, if no fallback exists the empty Locale::noCode.
Definition: Locale.cc:208
void downloadMediaInfo(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progressrcv)
Downloads the media info (/media.1) to a local directory.
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:28
OnMediaLocation & setChecksum(CheckSum val_r)
Set the checksum.
What is known about a repository.
Definition: RepoInfo.h:71
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Downloader.h:68
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Content file parser consumer.
Definition: Downloader.cc:213
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: progressdata.h:139
bool compatibleWith(const Arch &targetArch_r) const
Compatibility relation.
Definition: Arch.cc:505
LocaleSet repoRefreshLocales() const
List of locales for which translated package descriptions should be downloaded.
Definition: ZConfig.cc:1053
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition: Fetcher.cc:908
Downloader for SUSETags (YaST2) repositories Encapsulates all the knowledge of which files have to be...
Definition: Downloader.h:34
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:281
#define WAR
Definition: Logger.h:97
Parse repoindex part from a content file.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:861
const RepoInfo & repoInfo() const
Definition: Downloader.h:63
void setRepoIndexConsumer(const RepoIndexConsumer &fnc_r)
Consumer to call when repo index was parsed.
static Pathname search_deltafile(const Pathname &dir, const Pathname &file)
Definition: Downloader.cc:45
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition: String.h:1092
OnMediaLocation & setDeltafile(Pathname path)
Set the deltafile.
&#39;Language[_Country]&#39; codes.
Definition: Locale.h:49
Regular expression match result.
Definition: Regex.h:167
Base class for Exception.
Definition: Exception.h:145
void download(MediaSetAccess &media, const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc()) override
Download metadata to a local directory.
Definition: Downloader.cc:55
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
Arch systemArchitecture() const
The system architecture zypp uses.
Definition: ZConfig.cc:875
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
regex ZYPP_STR_REGEX regex ZYPP_STR_REGEX
Definition: Regex.h:70
Track changing files or directories.
Definition: RepoStatus.h:40
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:29
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1027
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.