Line data Source code
1 : /********************************************************************\
2 : * gnc-quotes.hpp -- proxy for Finance::Quote *
3 : * Copyright (C) 2021 Geert Janssens <geert@kobaltwit.be> *
4 : * *
5 : * This program is free software; you can redistribute it and/or *
6 : * modify it under the terms of the GNU General Public License as *
7 : * published by the Free Software Foundation; either version 2 of *
8 : * the License, or (at your option) any later version. *
9 : * *
10 : * This program is distributed in the hope that it will be useful, *
11 : * but WITHOUT ANY WARRANTY; without even the implied warranty of *
12 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13 : * GNU General Public License for more details. *
14 : * *
15 : * You should have received a copy of the GNU General Public License*
16 : * along with this program; if not, contact: *
17 : * *
18 : * Free Software Foundation Voice: +1-617-542-5942 *
19 : * 51 Franklin Street, Fifth Floor Fax: +1-617-542-2652 *
20 : * Boston, MA 02110-1301, USA gnu@gnu.org *
21 : \ *******************************************************************/
22 :
23 : #include <boost/process/environment.hpp>
24 : #include <config.h>
25 : #include <qoflog.h>
26 :
27 : #include <algorithm>
28 : #include <stdexcept>
29 : #include <vector>
30 : #include <string>
31 : #include <iostream>
32 : #include <boost/version.hpp>
33 : #if BOOST_VERSION < 107600
34 : // json_parser uses a deprecated version of bind.hpp
35 : #define BOOST_BIND_GLOBAL_PLACEHOLDERS
36 : #endif
37 : #include <boost/algorithm/string.hpp>
38 : #include <boost/filesystem.hpp>
39 : #include <boost/version.hpp>
40 : #if BOOST_VERSION < 108600
41 : #include <boost/process.hpp>
42 : #ifdef BOOST_WINDOWS_API
43 : #include <boost/process/windows.hpp>
44 : #endif
45 : #else
46 : #include <boost/process/v1/async.hpp>
47 : #include <boost/process/v1/child.hpp>
48 : #include <boost/process/v1/env.hpp>
49 : #include <boost/process/v1/environment.hpp>
50 : #include <boost/process/v1/error.hpp>
51 : #include <boost/process/v1/group.hpp>
52 : #include <boost/process/v1/io.hpp>
53 : #include <boost/process/v1/pipe.hpp>
54 : #include <boost/process/v1/search_path.hpp>
55 : #include <boost/process/v1/start_dir.hpp>
56 : #ifdef BOOST_WINDOWS_API
57 : #include <boost/process/v1/windows.hpp>
58 : #endif
59 : #endif
60 : #include <boost/regex.hpp>
61 : #include <boost/property_tree/ptree.hpp>
62 : #include <boost/property_tree/json_parser.hpp>
63 : #include <boost/iostreams/device/array.hpp>
64 : #include <boost/iostreams/stream_buffer.hpp>
65 : #include <boost/locale.hpp>
66 : #include <boost/asio.hpp>
67 : #include <glib.h>
68 : #include "gnc-commodity.hpp"
69 : #include <gnc-datetime.hpp>
70 : #include <gnc-numeric.hpp>
71 : #include "gnc-quotes.hpp"
72 :
73 : #include <gnc-commodity.h>
74 : #include <gnc-path.h>
75 : #include "gnc-ui-util.h"
76 : #include <gnc-prefs.h>
77 : #include <gnc-session.h>
78 : #include <regex.h>
79 : #include <qofbook.h>
80 :
81 : static const QofLogModule log_module = "gnc.price-quotes";
82 : static const char* av_api_env = "ALPHAVANTAGE_API_KEY";
83 : static const char* av_api_key = "alphavantage-api-key";
84 : static const char* yh_api_env = "FINANCEAPI_API_KEY";
85 : static const char* yh_api_key = "yhfinance-api-key";
86 :
87 : namespace bl = boost::locale;
88 : #if BOOST_VERSION < 108800
89 : namespace bp = boost::process;
90 : #else
91 : namespace bp = boost::process::v1;
92 : #endif
93 : namespace bfs = boost::filesystem;
94 : namespace bpt = boost::property_tree;
95 : namespace bio = boost::iostreams;
96 :
97 : using QuoteResult = std::tuple<int, StrVec, StrVec>;
98 :
99 : struct GncQuoteSourceError : public std::runtime_error
100 : {
101 0 : GncQuoteSourceError(const std::string& err) : std::runtime_error(err) {}
102 : };
103 :
104 : CommVec
105 : gnc_quotes_get_quotable_commodities(const gnc_commodity_table * table);
106 :
107 : class GncQuoteSource
108 : {
109 : public:
110 0 : virtual ~GncQuoteSource() = default;
111 : virtual const StrVec& get_sources() const noexcept = 0;
112 : virtual const std::string & get_version() const noexcept = 0;
113 : virtual QuoteResult get_quotes(const std::string& json_str) const = 0;
114 : };
115 :
116 :
117 : class GncQuotesImpl
118 : {
119 : public:
120 : // Constructor - checks for presence of Finance::Quote and import version and quote sources
121 : GncQuotesImpl ();
122 : explicit GncQuotesImpl (QofBook *book);
123 : GncQuotesImpl(QofBook*, std::unique_ptr<GncQuoteSource>);
124 :
125 : void fetch (QofBook *book);
126 : void fetch (CommVec& commodities);
127 : void fetch (gnc_commodity *comm);
128 : void report (const char* source, const StrVec& commodities, bool verbose);
129 :
130 0 : const std::string& version() noexcept { return m_quotesource->get_version(); }
131 0 : const QuoteSources& sources() noexcept { return m_sources; }
132 0 : bool had_failures() noexcept { return !m_failures.empty(); }
133 : const QFVec& failures() noexcept;
134 : std::string report_failures() noexcept;
135 :
136 : private:
137 : std::string query_fq (const char* source, const StrVec& commoditites);
138 : std::string query_fq (const CommVec&);
139 : bpt::ptree parse_quotes (const std::string& quote_str);
140 : void create_quotes(const bpt::ptree& pt, const CommVec& comm_vec);
141 : std::string comm_vec_to_json_string(const CommVec&) const;
142 : GNCPrice* parse_one_quote(const bpt::ptree&, gnc_commodity*);
143 :
144 : std::unique_ptr<GncQuoteSource> m_quotesource;
145 : QuoteSources m_sources;
146 : QFVec m_failures;
147 : QofBook *m_book;
148 : gnc_commodity *m_dflt_curr;
149 : };
150 :
151 : class GncFQQuoteSource final : public GncQuoteSource
152 : {
153 : const bfs::path c_cmd;
154 : std::string c_fq_wrapper;
155 : std::string m_version;
156 : StrVec m_sources;
157 : bp::environment m_env;
158 : public:
159 : GncFQQuoteSource();
160 0 : ~GncFQQuoteSource() = default;
161 0 : const std::string& get_version() const noexcept override { return m_version; }
162 0 : const StrVec& get_sources() const noexcept override { return m_sources; }
163 : QuoteResult get_quotes(const std::string&) const override;
164 : private:
165 : QuoteResult run_cmd (const StrVec& args, const std::string& json_string) const;
166 : void set_api_key(const char* api_pref, const char* api_env);
167 : };
168 :
169 : static void show_quotes(const bpt::ptree& pt, const StrVec& commodities, bool verbose);
170 : static void show_currency_quotes(const bpt::ptree& pt, const StrVec& commodities, bool verbose);
171 : static std::string parse_quotesource_error(const std::string& line);
172 :
173 : static const std::string empty_string{};
174 :
175 0 : GncFQQuoteSource::GncFQQuoteSource() :
176 0 : c_cmd{bp::search_path("perl")},
177 0 : m_version{}, m_sources{}, m_env{boost::this_process::environment()}
178 : {
179 0 : char *bindir = gnc_path_get_bindir();
180 0 : c_fq_wrapper = std::string(bindir) + "/finance-quote-wrapper";
181 0 : g_free(bindir);
182 0 : StrVec args{"-w", c_fq_wrapper, "-v"};
183 0 : auto [rv, sources, errors] = run_cmd(args, empty_string);
184 0 : if (rv)
185 : {
186 0 : std::string err{bl::translate("Failed to initialize Finance::Quote: ")};
187 0 : for (const auto& err_line : errors)
188 0 : err += err_line.empty() ? "" : err_line + "\n";
189 0 : throw(GncQuoteSourceError(err));
190 0 : }
191 0 : if (!errors.empty())
192 : {
193 0 : std::string err{bl::translate("Finance::Quote check returned error ")};
194 0 : for(const auto& err_line : errors)
195 0 : err += err.empty() ? "" : err_line + "\n";
196 0 : throw(GncQuoteSourceError(err));
197 0 : }
198 0 : auto version{sources.front()};
199 0 : if (version.empty())
200 : {
201 0 : std::string err{bl::translate("No Finance::Quote Version")};
202 0 : throw(GncQuoteSourceError(err));
203 0 : }
204 0 : m_version = std::move(version);
205 0 : sources.erase(sources.begin());
206 0 : m_sources = std::move(sources);
207 0 : std::sort (m_sources.begin(), m_sources.end());
208 :
209 0 : set_api_key(av_api_key, av_api_env);
210 0 : set_api_key(yh_api_key, yh_api_env);
211 0 : }
212 :
213 : QuoteResult
214 0 : GncFQQuoteSource::get_quotes(const std::string& json_str) const
215 : {
216 0 : StrVec args{"-w", c_fq_wrapper, "-f" };
217 0 : return run_cmd(args, json_str);
218 0 : }
219 :
220 : QuoteResult
221 0 : GncFQQuoteSource::run_cmd (const StrVec& args, const std::string& json_string) const
222 : {
223 0 : StrVec out_vec, err_vec;
224 : int cmd_result;
225 :
226 : try
227 : {
228 0 : std::future<std::vector<char> > out_buf, err_buf;
229 0 : boost::asio::io_context svc;
230 :
231 0 : auto input_buf = bp::buffer (json_string);
232 0 : bp::child process;
233 0 : process = bp::child(c_cmd, args,
234 0 : bp::std_out > out_buf,
235 0 : bp::std_err > err_buf,
236 0 : bp::std_in < input_buf,
237 : #ifdef BOOST_WINDOWS_API
238 : bp::windows::create_no_window,
239 : #endif
240 0 : m_env,
241 0 : svc);
242 :
243 0 : svc.run();
244 0 : process.wait();
245 :
246 : {
247 0 : auto raw = out_buf.get();
248 0 : std::vector<std::string> data;
249 0 : std::string line;
250 0 : bio::stream_buffer<bio::array_source> sb(raw.data(), raw.size());
251 0 : std::istream is(&sb);
252 :
253 0 : while (std::getline(is, line) && !line.empty())
254 : {
255 : #ifdef __WIN32
256 : if (line.back() == '\r')
257 : line.pop_back();
258 : #endif
259 0 : out_vec.push_back (std::move(line));
260 : }
261 0 : raw = err_buf.get();
262 0 : bio::stream_buffer<bio::array_source> eb(raw.data(), raw.size());
263 0 : std::istream es(&eb);
264 :
265 0 : while (std::getline(es, line) && !line.empty())
266 0 : err_vec.push_back (std::move(line));
267 0 : }
268 0 : cmd_result = process.exit_code();
269 0 : }
270 0 : catch (std::exception &e)
271 : {
272 0 : cmd_result = -1;
273 0 : err_vec.push_back(e.what());
274 0 : };
275 :
276 0 : return QuoteResult (cmd_result, std::move(out_vec), std::move(err_vec));
277 0 : }
278 :
279 : void
280 0 : GncFQQuoteSource::set_api_key(const char* api_key, const char* api_env)
281 : {
282 0 : auto key = gnc_prefs_get_string("general.finance-quote", api_key);
283 0 : if (key && *key)
284 : {
285 0 : m_env[api_env] = key;
286 0 : g_free(key);
287 : }
288 : else
289 : {
290 0 : if (api_key == av_api_key && m_env.find(api_env) == m_env.end())
291 0 : PWARN("No Alpha Vantage API key set, currency quotes and other "
292 : "AlphaVantage based quotes won't work.");
293 0 : g_free(key);
294 : }
295 0 : }
296 :
297 : /* GncQuotes implementation */
298 0 : GncQuotesImpl::GncQuotesImpl() : m_quotesource{new GncFQQuoteSource},
299 0 : m_sources{}, m_failures{},
300 0 : m_book{qof_session_get_book(gnc_get_current_session())},
301 0 : m_dflt_curr{gnc_default_currency()}
302 : {
303 0 : m_sources = m_quotesource->get_sources();
304 0 : }
305 :
306 0 : GncQuotesImpl::GncQuotesImpl(QofBook* book) : m_quotesource{new GncFQQuoteSource},
307 0 : m_sources{}, m_book{book},
308 0 : m_dflt_curr{gnc_default_currency()}
309 : {
310 0 : m_sources = m_quotesource->get_sources();
311 0 : }
312 :
313 0 : GncQuotesImpl::GncQuotesImpl(QofBook* book, std::unique_ptr<GncQuoteSource> quote_source) :
314 0 : m_quotesource{std::move(quote_source)},
315 0 : m_sources{}, m_book{book}, m_dflt_curr{gnc_default_currency()}
316 : {
317 0 : m_sources = m_quotesource->get_sources();
318 0 : }
319 :
320 : void
321 0 : GncQuotesImpl::fetch (QofBook *book)
322 : {
323 0 : if (!book)
324 0 : throw (GncQuoteException(bl::translate("GncQuotes::Fetch called with no book.")));
325 : auto commodities = gnc_quotes_get_quotable_commodities (
326 0 : gnc_commodity_table_get_table (book));
327 0 : fetch (commodities);
328 0 : }
329 :
330 : void
331 0 : GncQuotesImpl::fetch (gnc_commodity *comm)
332 : {
333 0 : auto commodities = CommVec {comm};
334 0 : fetch (commodities);
335 0 : }
336 :
337 : void
338 0 : GncQuotesImpl::fetch (CommVec& commodities)
339 : {
340 0 : m_failures.clear();
341 0 : if (commodities.empty())
342 0 : throw (GncQuoteException(bl::translate("GncQuotes::Fetch called with no commodities.")));
343 0 : auto quote_str{query_fq (commodities)};
344 0 : auto ptree{parse_quotes (quote_str)};
345 0 : create_quotes(ptree, commodities);
346 0 : }
347 :
348 : void
349 0 : GncQuotesImpl::report (const char* source, const StrVec& commodities,
350 : bool verbose)
351 : {
352 0 : if (!source)
353 0 : throw (GncQuoteException(bl::translate("GncQuotes::Report called with no source.")));
354 :
355 0 : bool is_currency{strcmp(source, "currency") == 0};
356 0 : m_failures.clear();
357 0 : if (commodities.empty())
358 : {
359 0 : std::cerr << _("There were no commodities for which to retrieve quotes.") << std::endl;
360 0 : return;
361 : }
362 : try
363 : {
364 0 : auto quote_str{query_fq (source, commodities)};
365 0 : auto ptree{parse_quotes (quote_str)};
366 0 : auto source_pt_ai{ptree.find(source)};
367 0 : if (is_currency)
368 0 : show_currency_quotes(source_pt_ai->second, commodities, verbose);
369 : else
370 0 : show_quotes(source_pt_ai->second, commodities, verbose);
371 0 : }
372 0 : catch (const GncQuoteException& err)
373 : {
374 0 : std::cerr << _("Finance::Quote retrieval failed with error ") << err.what() << std::endl;
375 0 : }
376 : }
377 :
378 : const QFVec&
379 0 : GncQuotesImpl::failures() noexcept
380 : {
381 0 : return m_failures;
382 : }
383 :
384 : static std::string
385 0 : explain(GncQuoteError err, const std::string& errmsg)
386 : {
387 0 : std::string retval;
388 0 : switch (err)
389 : {
390 0 : case GncQuoteError::NO_RESULT:
391 0 : if (errmsg.empty())
392 0 : retval += _("Finance::Quote returned no data and set no error.");
393 : else
394 0 : retval += _("Finance::Quote returned an error: ") + errmsg;
395 0 : break;
396 0 : case GncQuoteError::QUOTE_FAILED:
397 0 : if (errmsg.empty())
398 0 : retval += _("Finance::Quote reported failure set no error.");
399 : else
400 0 : retval += _("Finance::Quote reported failure with error: ") + errmsg;
401 0 : break;
402 0 : case GncQuoteError::NO_CURRENCY:
403 0 : retval += _("Finance::Quote returned a quote with no currency.");
404 0 : break;
405 0 : case GncQuoteError::UNKNOWN_CURRENCY:
406 0 : retval += _("Finance::Quote returned a quote with a currency GnuCash doesn't know about.");
407 0 : break;
408 0 : case GncQuoteError::NO_PRICE:
409 0 : retval += _("Finance::Quote returned a quote with no price element.");
410 0 : break;
411 0 : case GncQuoteError::PRICE_PARSE_FAILURE:
412 0 : retval += _("Finance::Quote returned a quote with a price that GnuCash was unable to covert to a number.");
413 0 : break;
414 0 : case GncQuoteError::SUCCESS:
415 : default:
416 0 : retval += _("The quote has no error set.");
417 0 : break;
418 : }
419 0 : return retval;
420 0 : }
421 :
422 : std::string
423 0 : GncQuotesImpl::report_failures() noexcept
424 : {
425 0 : std::string retval{_("Quotes for the following commodities were unavailable or unusable:\n")};
426 0 : std::for_each(m_failures.begin(), m_failures.end(),
427 0 : [&retval](auto failure)
428 : {
429 0 : auto [ns, sym, reason, err] = failure;
430 0 : retval += "* " + ns + ":" + sym + " " +
431 0 : explain(reason, err) + "\n";
432 0 : });
433 0 : return retval;
434 : }
435 :
436 : /* **** Private function implementations ****/
437 :
438 : using Path = bpt::ptree::path_type;
439 0 : static inline Path make_quote_path(const std::string &name_space,
440 : const std::string &symbol)
441 : {
442 : using Path = bpt::ptree::path_type;
443 0 : Path key{name_space, '|'};
444 0 : key /= Path{symbol, '|'};
445 0 : return key;
446 0 : };
447 :
448 : std::string
449 0 : GncQuotesImpl::comm_vec_to_json_string(const CommVec &comm_vec) const
450 : {
451 0 : bpt::ptree pt, pt_child;
452 0 : pt.put("defaultcurrency", gnc_commodity_get_mnemonic(m_dflt_curr));
453 :
454 0 : std::for_each (comm_vec.cbegin(), comm_vec.cend(),
455 0 : [this, &pt] (auto comm)
456 : {
457 0 : auto comm_mnemonic = gnc_commodity_get_mnemonic (comm);
458 0 : auto comm_ns = std::string("currency");
459 0 : if (gnc_commodity_is_currency (comm))
460 : {
461 0 : if (gnc_commodity_equiv(comm, m_dflt_curr) ||
462 0 : (!comm_mnemonic || (strcmp(comm_mnemonic, "XXX") == 0)))
463 0 : return;
464 : }
465 : else
466 0 : comm_ns = gnc_quote_source_get_internal_name(gnc_commodity_get_quote_source(comm));
467 :
468 0 : pt.put (make_quote_path(comm_ns, comm_mnemonic), "");
469 0 : }
470 : );
471 :
472 0 : std::ostringstream result;
473 0 : bpt::write_json(result, pt);
474 0 : return result.str();
475 0 : }
476 :
477 : static inline std::string
478 0 : get_quotes(const std::string& json_str, const std::unique_ptr<GncQuoteSource>& qs)
479 : {
480 0 : auto [rv, quotes, errors] = qs->get_quotes(json_str);
481 0 : std::string answer;
482 :
483 0 : if (rv == 0)
484 : {
485 0 : for (const auto& line : quotes)
486 0 : answer.append(line + "\n");
487 : }
488 : else
489 : {
490 0 : std::string err_str;
491 0 : for (const auto& line: errors)
492 : {
493 0 : if (line == "invalid_json\n")
494 0 : PERR("Finanace Quote Wrapper was unable to parse %s",
495 : json_str.c_str());
496 0 : err_str += parse_quotesource_error(line);
497 : }
498 0 : throw(GncQuoteException(err_str));
499 0 : }
500 :
501 0 : return answer;
502 0 : }
503 :
504 : std::string
505 0 : GncQuotesImpl::query_fq (const char* source, const StrVec& commodities)
506 : {
507 0 : bpt::ptree pt;
508 0 : auto is_currency{strcmp(source, "currency") == 0};
509 :
510 0 : if (is_currency && commodities.size() < 2)
511 0 : throw(GncQuoteException(_("Currency quotes requires at least two currencies")));
512 :
513 0 : if (is_currency)
514 0 : pt.put("defaultcurrency", commodities[0].c_str());
515 : else
516 0 : pt.put("defaultcurrency", gnc_commodity_get_mnemonic(m_dflt_curr));
517 :
518 0 : std::for_each(is_currency ? ++commodities.cbegin() : commodities.cbegin(),
519 : commodities.cend(),
520 0 : [source, &pt](auto sym)
521 : {
522 0 : pt.put(make_quote_path(source, sym), "");
523 0 : });
524 0 : std::ostringstream result;
525 0 : bpt::write_json(result, pt);
526 0 : auto result_str{result.str()};
527 0 : PINFO("Query JSON: %s\n", result_str.c_str());
528 0 : return get_quotes(result.str(), m_quotesource);
529 0 : }
530 :
531 : std::string
532 0 : GncQuotesImpl::query_fq (const CommVec& comm_vec)
533 : {
534 0 : auto json_str{comm_vec_to_json_string(comm_vec)};
535 0 : PINFO("Query JSON: %s\n", json_str.c_str());
536 0 : return get_quotes(json_str, m_quotesource);
537 0 : }
538 :
539 : struct PriceParams
540 : {
541 : const char* ns;
542 : const char* mnemonic;
543 : bool success;
544 : std::string type;
545 : boost::optional<std::string> price;
546 : bool inverted;
547 : boost::optional<std::string> date;
548 : boost::optional<std::string> time;
549 : boost::optional<std::string> currency;
550 : boost::optional<std::string> errormsg;
551 : };
552 :
553 : static void
554 0 : get_price_and_type(PriceParams& p, const bpt::ptree& comm_pt)
555 : {
556 0 : p.type = "last";
557 0 : p.price = comm_pt.get_optional<std::string> (p.type);
558 :
559 0 : if (!p.price)
560 : {
561 0 : p.type = "nav";
562 0 : p.price = comm_pt.get_optional<std::string> (p.type);
563 : }
564 :
565 0 : if (!p.price)
566 : {
567 0 : p.type = "price";
568 0 : p.price = comm_pt.get_optional<std::string> (p.type);
569 : /* guile wrapper used "unknown" as price type when "price" was found,
570 : * reproducing here to keep same result for users in the pricedb */
571 0 : p.type = p.price ? "unknown" : "missing";
572 : }
573 0 : }
574 :
575 : static void
576 0 : parse_quote_json(PriceParams& p, const bpt::ptree& comm_pt)
577 : {
578 0 : auto success = comm_pt.get_optional<bool> ("success");
579 0 : p.success = success && *success;
580 0 : if (!p.success)
581 0 : p.errormsg = comm_pt.get_optional<std::string> ("errormsg");
582 0 : get_price_and_type(p, comm_pt);
583 0 : auto inverted = comm_pt.get_optional<bool> ("inverted");
584 0 : p.inverted = inverted && *inverted;
585 0 : p.date = comm_pt.get_optional<std::string> ("date");
586 0 : p.time = comm_pt.get_optional<std::string> ("time");
587 0 : p.currency = comm_pt.get_optional<std::string> ("currency");
588 :
589 :
590 0 : PINFO("Commodity: %s", p.mnemonic);
591 0 : PINFO(" Success: %s", (p.success ? "yes" : "no"));
592 0 : PINFO(" Date: %s", (p.date ? p.date->c_str() : "missing"));
593 0 : PINFO(" Time: %s", (p.time ? p.time->c_str() : "missing"));
594 0 : PINFO(" Currency: %s", (p.currency ? p.currency->c_str() : "missing"));
595 0 : PINFO(" Price: %s", (p.price ? p.price->c_str() : "missing"));
596 0 : PINFO(" Inverted: %s\n", (p.inverted ? "yes" : "no"));
597 0 : }
598 :
599 : static time64
600 0 : calc_price_time(const PriceParams& p)
601 : {
602 : /* Note that as of F::Q v. 1.52 the only sources that provide
603 : * quote times are ftfunds (aka ukfunds), morningstarch, and
604 : * mstaruk_fund, but it's faked with a comment "Set a dummy time
605 : * as gnucash insists on having a valid format". It's also wrong,
606 : * as it lacks seconds. Best ignored.
607 : */
608 0 : if (p.date && !p.date->empty())
609 : {
610 : try
611 : {
612 0 : auto quote_time{GncDateTime(GncDate(*p.date, "m-d-y"))};
613 0 : PINFO("Quote date included, using %s for %s:%s",
614 : quote_time.format("%Y-%m-%d %H:%M:%S %z").c_str(), p.ns, p.mnemonic);
615 0 : return static_cast<time64>(quote_time);
616 0 : }
617 0 : catch (const std::exception &err)
618 : {
619 0 : auto now{GncDateTime()};
620 0 : PWARN("Warning: failed to parse quote date '%s' for %s:%s because %s - will use %s",
621 : p.date->c_str(), p.ns, p.mnemonic, err.what(), now.format("%Y-%m-%d %H:%M:%S %z").c_str());
622 0 : return static_cast<time64>(now);
623 0 : }
624 : }
625 :
626 0 : auto now{GncDateTime()};
627 0 : PINFO("No date was returned for %s:%s - will use %s",
628 : p.ns, p.mnemonic, now.format("%Y-%m-%d %H:%M:%S %z").c_str());
629 0 : return static_cast<time64>(now);
630 0 : }
631 :
632 : static boost::optional<GncNumeric>
633 0 : get_price(const PriceParams& p)
634 : {
635 0 : boost::optional<GncNumeric> price;
636 : try
637 : {
638 0 : price = GncNumeric { *p.price };
639 : }
640 0 : catch (...)
641 : {
642 0 : PWARN("Skipped %s:%s - failed to parse returned price '%s'",
643 : p.ns, p.mnemonic, p.price->c_str());
644 0 : }
645 :
646 0 : if (price && p.inverted)
647 0 : *price = price->inv();
648 :
649 0 : return price;
650 0 : }
651 :
652 : static gnc_commodity*
653 0 : get_currency(const PriceParams& p, QofBook* book, QFVec& failures)
654 : {
655 0 : if (!p.currency)
656 : {
657 0 : failures.emplace_back(p.ns, p.mnemonic, GncQuoteError::NO_CURRENCY,
658 : empty_string);
659 0 : PWARN("Skipped %s:%s - Finance::Quote returned a quote with no currency",
660 : p.ns, p.mnemonic);
661 0 : return nullptr;
662 : }
663 0 : std::string curr_str = *p.currency;
664 0 : boost::to_upper (curr_str);
665 0 : auto commodity_table = gnc_commodity_table_get_table (book);
666 0 : auto currency = gnc_commodity_table_lookup (commodity_table, "ISO4217", curr_str.c_str());
667 :
668 0 : if (!currency)
669 : {
670 0 : failures.emplace_back(p.ns, p.mnemonic,
671 0 : GncQuoteError::UNKNOWN_CURRENCY, empty_string);
672 0 : PWARN("Skipped %s:%s - failed to parse returned currency '%s'",
673 : p.ns, p.mnemonic, p.currency->c_str());
674 0 : return nullptr;
675 : }
676 :
677 0 : return currency;
678 0 : }
679 :
680 : GNCPrice*
681 0 : GncQuotesImpl::parse_one_quote(const bpt::ptree& pt, gnc_commodity* comm)
682 : {
683 0 : PriceParams p;
684 0 : bpt::ptree comm_pt;
685 :
686 0 : p.ns = gnc_commodity_get_namespace (comm);
687 0 : p.mnemonic = gnc_commodity_get_mnemonic (comm);
688 0 : if (gnc_commodity_equiv(comm, m_dflt_curr) ||
689 0 : (!p.mnemonic || (strcmp (p.mnemonic, "XXX") == 0)))
690 0 : return nullptr;
691 0 : auto source{gnc_quote_source_get_internal_name(gnc_commodity_get_quote_source(comm))};
692 0 : auto source_pt_ai{pt.find(source)};
693 0 : auto ok{source_pt_ai != pt.not_found()};
694 0 : if (ok)
695 : {
696 0 : auto comm_pt_ai{source_pt_ai->second.find(p.mnemonic)};
697 0 : ok = (comm_pt_ai != pt.not_found());
698 0 : if (ok)
699 0 : comm_pt = comm_pt_ai->second;
700 : }
701 0 : if (!ok)
702 : {
703 0 : m_failures.emplace_back(p.ns, p.mnemonic, GncQuoteError::NO_RESULT,
704 : empty_string);
705 0 : PINFO("Skipped %s:%s - Finance::Quote didn't return any data from %s.",
706 : p.ns, p.mnemonic, source);
707 0 : return nullptr;
708 : }
709 :
710 0 : parse_quote_json(p, comm_pt);
711 0 : if (!p.success)
712 : {
713 0 : m_failures.emplace_back(p.ns, p.mnemonic, GncQuoteError::QUOTE_FAILED,
714 0 : p.errormsg ? *p.errormsg : empty_string);
715 0 : PWARN("Skipped %s:%s - Finance::Quote returned fetch failure.\nReason %s",
716 : p.ns, p.mnemonic,
717 : (p.errormsg ? p.errormsg->c_str() : "unknown"));
718 0 : return nullptr;
719 : }
720 :
721 0 : if (!p.price)
722 : {
723 0 : m_failures.emplace_back(p.ns, p.mnemonic,
724 0 : GncQuoteError::NO_PRICE, empty_string);
725 0 : PWARN("Skipped %s:%s - Finance::Quote didn't return a valid price",
726 : p.ns, p.mnemonic);
727 0 : return nullptr;
728 : }
729 :
730 0 : auto price{get_price(p)};
731 0 : if (!price)
732 : {
733 0 : m_failures.emplace_back(p.ns, p.mnemonic,
734 0 : GncQuoteError::PRICE_PARSE_FAILURE,
735 : empty_string);
736 0 : return nullptr;
737 : }
738 :
739 0 : auto currency{get_currency(p, m_book, m_failures)};
740 0 : if (!currency)
741 0 : return nullptr;
742 :
743 0 : auto quotedt{calc_price_time(p)};
744 0 : auto gnc_price = gnc_price_create (m_book);
745 0 : gnc_price_begin_edit (gnc_price);
746 0 : gnc_price_set_commodity (gnc_price, comm);
747 0 : gnc_price_set_currency (gnc_price, currency);
748 0 : gnc_price_set_time64 (gnc_price, static_cast<time64> (quotedt));
749 0 : gnc_price_set_source (gnc_price, PRICE_SOURCE_FQ);
750 0 : gnc_price_set_typestr (gnc_price, p.type.c_str());
751 0 : gnc_price_set_value (gnc_price, *price);
752 0 : gnc_price_commit_edit (gnc_price);
753 0 : return gnc_price;
754 0 : }
755 :
756 : bpt::ptree
757 0 : GncQuotesImpl::parse_quotes (const std::string& quote_str)
758 : {
759 0 : bpt::ptree pt;
760 0 : std::istringstream ss {quote_str};
761 0 : std::string what;
762 :
763 : try
764 : {
765 0 : bpt::read_json (ss, pt);
766 : }
767 0 : catch (bpt::json_parser_error &e) {
768 0 : what = e.what();
769 0 : }
770 0 : catch (const std::runtime_error& e)
771 : {
772 0 : what = e.what();
773 0 : }
774 0 : catch (const std::logic_error& e)
775 : {
776 0 : what = e.what();
777 0 : }
778 0 : catch (...) {
779 0 : std::string error_msg{_("Failed to parse result returned by Finance::Quote.")};
780 0 : error_msg += "\n";
781 : //Translators: This labels the return value of a query to Finance::Quote written in an error.
782 0 : error_msg += _("Result:");
783 0 : error_msg += "\n";
784 0 : error_msg += quote_str;
785 0 : throw(GncQuoteException(error_msg));
786 0 : }
787 0 : if (!what.empty())
788 : {
789 0 : std::string error_msg{_("Failed to parse result returned by Finance::Quote.")};
790 0 : error_msg += "\n";
791 : //Translators: This is the error message reported by the Online Quotes processing code.
792 0 : error_msg += _("Error message:");
793 0 : error_msg += "\n";
794 0 : error_msg += what;
795 0 : error_msg += "\n";
796 : //Translators: This labels the return value of a query to Finance::Quote written in an error.
797 0 : error_msg += _("Result:");
798 0 : error_msg += "\n";
799 0 : error_msg += quote_str;
800 0 : throw(GncQuoteException(error_msg));
801 0 : }
802 0 : return pt;
803 0 : }
804 :
805 : void
806 0 : GncQuotesImpl::create_quotes (const bpt::ptree& pt, const CommVec& comm_vec)
807 : {
808 0 : auto pricedb{gnc_pricedb_get_db(m_book)};
809 0 : for (auto comm : comm_vec)
810 : {
811 0 : auto price{parse_one_quote(pt, comm)};
812 0 : if (!price)
813 0 : continue;
814 : // See the comment at gnc_pricedb_add_price
815 0 : gnc_pricedb_add_price(pricedb, price);
816 : }
817 0 : }
818 :
819 : static void
820 0 : show_verbose_quote(const bpt::ptree& comm_pt)
821 : {
822 0 : std::for_each(comm_pt.begin(), comm_pt.end(),
823 0 : [](auto elem) {
824 0 : std::cout << std::setw(12) << std::right << elem.first << " => " <<
825 0 : std::left << elem.second.data() << "\n";
826 0 : });
827 0 : std::cout << std::endl;
828 0 : }
829 :
830 : static void
831 0 : show_gnucash_quote(const bpt::ptree& comm_pt)
832 : {
833 0 : constexpr const char* ptr{"<=== "};
834 0 : constexpr const char* dptr{"<=\\ "};
835 0 : constexpr const char* uptr{"<=/ "};
836 : //Translators: Means that the preceding element is required
837 0 : const char* reqd{C_("Finance::Quote", "required")};
838 : //Translators: Means that the quote will work best if the preceding element is provided
839 0 : const char* rec{C_("Finance::Quote", "recommended")};
840 : //Translators: Means that one of the indicated elements is required
841 0 : const char* oot{C_("Finance::Quote", "one of these")};
842 : //Translators: Means that a required element wasn't reported. The *s are for emphasis.
843 0 : const char* miss{C_("Finance::Quote", "**missing**")};
844 :
845 0 : const std::string miss_str{miss};
846 0 : auto outline{[](const char* label, std::string value, const char* pointer, const char* req) {
847 0 : std::cout << std::setw(12) << std::right << label << std::setw(16) << std::left <<
848 0 : value << pointer << req << "\n";
849 0 : }};
850 0 : std::cout << _("Finance::Quote fields GnuCash uses:") << "\n";
851 : //Translators: The stock or Mutual Fund symbol, ISIN, CUSIP, etc.
852 0 : outline(C_("Finance::Quote", "symbol: "), comm_pt.get<char>("symbol", miss), ptr, reqd);
853 : //Translators: The date of the quote.
854 0 : outline(C_("Finance::Quote", "date: "), comm_pt.get<char>("date", miss), ptr, rec);
855 : //Translators: The quote currency
856 0 : outline(C_("Finance::Quote", "currency: "), comm_pt.get<char>("currency", miss), ptr, reqd);
857 0 : auto last{comm_pt.get<char>("last", "")};
858 0 : auto nav{comm_pt.get<char>("nav", "")};
859 0 : auto price{comm_pt.get<char>("nav", "")};
860 0 : auto no_price{last.empty() && nav.empty() && price.empty()};
861 : //Translators: The quote is for the most recent trade on the exchange
862 0 : outline(C_("Finance::Quote", "last: "), no_price ? miss_str : last, dptr, "");
863 : //Translators: The quote is for an open-ended mutual fund and represents the net asset value of one unit of the fund at the previous close of trading.
864 0 : outline(C_("Finance::Quote", "nav: "), no_price ? miss_str : nav, ptr, oot);
865 : //Translators: The quote is neither a last trade nor an NAV.
866 0 : outline(C_("Finance::Quote", "price: "), no_price ? miss_str : price, uptr, "");
867 0 : std::cout << std::endl;
868 0 : }
869 : static const bpt::ptree empty_tree{};
870 :
871 : static inline const bpt::ptree&
872 0 : get_commodity_data(const bpt::ptree& pt, const std::string& comm)
873 : {
874 0 : auto commdata{pt.find(comm)};
875 0 : if (commdata == pt.not_found())
876 : {
877 0 : std::cout << comm << " " << _("Finance::Quote returned no data and set no error.") << std::endl;
878 0 : return empty_tree;
879 : }
880 0 : auto& comm_pt{commdata->second};
881 0 : auto success = comm_pt.get_optional<bool> ("success");
882 0 : if (!(success && *success))
883 : {
884 0 : auto errormsg = comm_pt.get_optional<std::string> ("errormsg");
885 0 : if (errormsg && !errormsg->empty())
886 0 : std::cout << _("Finance::Quote reported a failure for symbol ") <<
887 0 : comm << ": " << *errormsg << std::endl;
888 : else
889 0 : std::cout << _("Finance::Quote failed silently to retrieve a quote for symbol ") <<
890 0 : comm << std::endl;
891 0 : return empty_tree;
892 0 : }
893 0 : return comm_pt;
894 : }
895 :
896 : static void
897 0 : show_quotes(const bpt::ptree& pt, const StrVec& commodities, bool verbose)
898 : {
899 0 : for (const auto& comm : commodities)
900 : {
901 0 : auto comm_pt{get_commodity_data(pt, comm)};
902 :
903 0 : if (comm_pt == empty_tree)
904 0 : continue;
905 :
906 0 : if (verbose)
907 : {
908 0 : std::cout << comm << ":\n";
909 0 : show_verbose_quote(comm_pt);
910 : }
911 : else
912 : {
913 0 : show_gnucash_quote(comm_pt);
914 : }
915 0 : }
916 0 : }
917 :
918 : static void
919 0 : show_currency_quotes(const bpt::ptree& pt, const StrVec& commodities, bool verbose)
920 : {
921 0 : auto to_cur{commodities.front()};
922 0 : for (const auto& comm : commodities)
923 : {
924 0 : if (comm == to_cur)
925 0 : continue;
926 :
927 0 : auto comm_pt{get_commodity_data(pt, comm)};
928 :
929 0 : if (comm_pt == empty_tree)
930 0 : continue;
931 :
932 0 : if (verbose)
933 : {
934 0 : std::cout << comm << ":\n";
935 0 : show_verbose_quote(comm_pt);
936 : }
937 : else
938 : {
939 : std::cout << "1 " << comm << " = " <<
940 0 : comm_pt.get<char>("last", "Not Found") << " " << to_cur << "\n";
941 : }
942 0 : std::cout << std::endl;
943 0 : }
944 0 : }
945 :
946 : static std::string
947 0 : parse_quotesource_error(const std::string& line)
948 : {
949 0 : std::string err_str;
950 0 : if (line == "invalid_json\n")
951 : {
952 0 : err_str += _("GnuCash submitted invalid json to Finance::Quote. The details were logged.");
953 : }
954 0 : else if (line.substr(0, 15) == "missing_modules")
955 : {
956 0 : PERR("Missing Finance::Quote Dependencies: %s",
957 : line.substr(17).c_str());
958 0 : err_str += _("Perl is missing the following modules. Please see https://wiki.gnucash.org/wiki/Online_Quotes#Finance::Quote for detailed corrective action. ");
959 0 : err_str += line.substr(17);
960 : }
961 : else
962 : {
963 0 : PERR("Unrecognized Finance::Quote Error %s", line.c_str());
964 0 : err_str +=_("Unrecognized Finance::Quote Error: ");
965 0 : err_str += line;
966 : }
967 0 : err_str += "\n";
968 0 : return err_str;
969 0 : }
970 :
971 : /********************************************************************
972 : * gnc_quotes_get_quotable_commodities
973 : * list commodities in a given namespace that get price quotes
974 : ********************************************************************/
975 : /* Helper function to be passed to g_list_for_each applied to the result
976 : * of gnc_commodity_namespace_get_commodity_list.
977 : */
978 : static void
979 0 : get_quotables_helper1 (gpointer value, gpointer data)
980 : {
981 0 : auto l = static_cast<CommVec *> (data);
982 0 : auto comm = static_cast<gnc_commodity *> (value);
983 0 : auto quote_flag = gnc_commodity_get_quote_flag (comm);
984 0 : auto quote_source = gnc_commodity_get_quote_source (comm);
985 0 : auto quote_source_supported = gnc_quote_source_get_supported (quote_source);
986 :
987 0 : if (!quote_flag ||
988 0 : !quote_source || !quote_source_supported)
989 0 : return;
990 0 : l->push_back (comm);
991 : }
992 :
993 : // Helper function to be passed to gnc_commodity_table_for_each
994 : static gboolean
995 0 : get_quotables_helper2 (gnc_commodity *comm, gpointer data)
996 : {
997 0 : auto l = static_cast<CommVec *> (data);
998 0 : auto quote_flag = gnc_commodity_get_quote_flag (comm);
999 0 : auto quote_source = gnc_commodity_get_quote_source (comm);
1000 0 : auto quote_source_supported = gnc_quote_source_get_supported (quote_source);
1001 :
1002 0 : if (!quote_flag ||
1003 0 : !quote_source || !quote_source_supported)
1004 0 : return TRUE;
1005 0 : l->push_back (comm);
1006 0 : return TRUE;
1007 : }
1008 :
1009 : CommVec
1010 0 : gnc_quotes_get_quotable_commodities (const gnc_commodity_table * table)
1011 : {
1012 0 : gnc_commodity_namespace * ns = NULL;
1013 : const char *name_space;
1014 : GList * nslist, * tmp;
1015 0 : CommVec l;
1016 : regex_t pattern;
1017 0 : const char *expression = gnc_prefs_get_namespace_regexp ();
1018 :
1019 : // ENTER("table=%p, expression=%s", table, expression);
1020 0 : if (!table)
1021 0 : return CommVec ();
1022 :
1023 0 : if (expression && *expression)
1024 : {
1025 0 : if (regcomp (&pattern, expression, REG_EXTENDED | REG_ICASE) != 0)
1026 : {
1027 : // LEAVE ("Cannot compile regex");
1028 0 : return CommVec ();
1029 : }
1030 :
1031 0 : nslist = gnc_commodity_table_get_namespaces (table);
1032 0 : for (tmp = nslist; tmp; tmp = tmp->next)
1033 : {
1034 0 : name_space = static_cast<const char *> (tmp->data);
1035 0 : if (regexec (&pattern, name_space, 0, NULL, 0) == 0)
1036 : {
1037 : // DEBUG ("Running list of %s commodities", name_space);
1038 0 : ns = gnc_commodity_table_find_namespace (table, name_space);
1039 0 : if (ns)
1040 : {
1041 0 : auto cm_list = gnc_commodity_namespace_get_commodity_list (ns);
1042 0 : g_list_foreach (cm_list, &get_quotables_helper1, (gpointer) &l);
1043 0 : g_list_free (cm_list);
1044 : }
1045 : }
1046 : }
1047 0 : g_list_free (nslist);
1048 0 : regfree (&pattern);
1049 0 : }
1050 : else
1051 : {
1052 0 : gnc_commodity_table_foreach_commodity (table, get_quotables_helper2,
1053 : (gpointer) &l);
1054 : }
1055 : //LEAVE ("list head %p", &l);
1056 0 : return l;
1057 0 : }
1058 :
1059 : /* Public interface functions */
1060 : // Constructor - checks for presence of Finance::Quote and import version and quote sources
1061 0 : GncQuotes::GncQuotes ()
1062 : {
1063 : try
1064 : {
1065 0 : m_impl = std::make_unique<GncQuotesImpl>();
1066 0 : } catch (const GncQuoteSourceError &err) {
1067 0 : throw(GncQuoteException(err.what()));
1068 0 : }
1069 0 : }
1070 :
1071 :
1072 : void
1073 0 : GncQuotes::fetch (QofBook *book)
1074 : {
1075 0 : m_impl->fetch (book);
1076 0 : }
1077 :
1078 0 : void GncQuotes::fetch (CommVec& commodities)
1079 : {
1080 0 : m_impl->fetch (commodities);
1081 0 : }
1082 :
1083 0 : void GncQuotes::fetch (gnc_commodity *comm)
1084 : {
1085 0 : m_impl->fetch (comm);
1086 0 : }
1087 :
1088 0 : void GncQuotes::report (const char* source, const StrVec& commodities,
1089 : bool verbose)
1090 : {
1091 0 : m_impl->report(source, commodities, verbose);
1092 0 : }
1093 :
1094 0 : const std::string& GncQuotes::version() noexcept
1095 : {
1096 0 : return m_impl->version ();
1097 : }
1098 :
1099 0 : const QuoteSources& GncQuotes::sources() noexcept
1100 : {
1101 0 : return m_impl->sources ();
1102 : }
1103 :
1104 0 : GncQuotes::~GncQuotes() = default;
1105 :
1106 : bool
1107 0 : GncQuotes::had_failures() noexcept
1108 : {
1109 0 : return m_impl->had_failures();
1110 : }
1111 :
1112 : const QFVec&
1113 0 : GncQuotes::failures() noexcept
1114 : {
1115 0 : return m_impl->failures();
1116 : }
1117 :
1118 : const std::string
1119 0 : GncQuotes::report_failures() noexcept
1120 : {
1121 0 : return m_impl->report_failures();
1122 : }
|
