/* GMM (C) 2018 Stephane Charette <stephanecharette@gmail.com>
 * $Id: TabCanvas.cpp 2657 2018-11-11 01:30:24Z stephane $
 */

#include "GMM.hpp"


TabCanvas::TabCanvas(const std::string & name) :
	Thread("thread for device prefix " + name),
	prefix(name),
	tab_is_active(false),
	export_image(cfg().get_bool(prefix + "_export_image")),
	timer_frequency_in_milliseconds(cfg().get_int(prefix + "_chart_redraw_ms")),
	comm_handle(INVALID_HANDLE_VALUE),
	run_sum(0.0),
	pkg_sum(0.0),
	run_average(0.0),
	pkg_average(0.0),
	run_std_dev(0.0),
	pkg_std_dev(0.0)
{
	const Colour pink_background(0xe6, 0xcc, 0xf2);	// total
	const Colour blue_background(0xc0, 0xc0, 0xff);	// local

	// ****************
	// top-right corner
	// ****************

	addAndMakeVisible(kiln_num_label				);
	addAndMakeVisible(kiln_num_editor				);
	addAndMakeVisible(run_label						);
	addAndMakeVisible(run_editor					);
	addAndMakeVisible(pkg_label						);
	addAndMakeVisible(pkg_editor					);
	addAndMakeVisible(total_number_of_boards_label	);
	addAndMakeVisible(total_number_of_boards_editor	);
	addAndMakeVisible(total_average_moisture_label	);
	addAndMakeVisible(total_average_moisture_editor	);
	addAndMakeVisible(total_std_dev_moisture_label	);
	addAndMakeVisible(total_std_dev_moisture_editor	);
	addAndMakeVisible(local_number_of_boards_label	);
	addAndMakeVisible(local_number_of_boards_editor	);
	addAndMakeVisible(local_average_moisture_label	);
	addAndMakeVisible(local_average_moisture_editor	);
	addAndMakeVisible(local_std_dev_moisture_label	);
	addAndMakeVisible(local_std_dev_moisture_editor	);

	kiln_num_label	.setText("Kiln:", NotificationType::sendNotification);
	run_label		.setText("Run:"	, NotificationType::sendNotification);
	pkg_label		.setText("Pkg:"	, NotificationType::sendNotification);
	kiln_num_label	.setJustificationType(Justification::centredRight);
	run_label		.setJustificationType(Justification::centredRight);
	pkg_label		.setJustificationType(Justification::centredRight);

	kiln_num_editor	.setReadOnly(true);		kiln_num_editor	.setJustification(Justification::centredRight);
	run_editor		.setReadOnly(true);		run_editor		.setJustification(Justification::centredRight);
	pkg_editor		.setReadOnly(true);		pkg_editor		.setJustification(Justification::centredRight);

	total_number_of_boards_label	.setText("Boards:"	, NotificationType::sendNotification);
	total_average_moisture_label	.setText("Avg:"		, NotificationType::sendNotification);
	total_std_dev_moisture_label	.setText("StDv:"	, NotificationType::sendNotification);
	total_number_of_boards_label	.setJustificationType(Justification::centredRight);
	total_average_moisture_label	.setJustificationType(Justification::centredRight);
	total_std_dev_moisture_label	.setJustificationType(Justification::centredRight);

	total_number_of_boards_editor	.setReadOnly(true);
	total_average_moisture_editor	.setReadOnly(true);
	total_std_dev_moisture_editor	.setReadOnly(true);
	total_number_of_boards_editor	.setJustification(Justification::centredRight);
	total_average_moisture_editor	.setJustification(Justification::centredRight);
	total_std_dev_moisture_editor	.setJustification(Justification::centredRight);
	total_number_of_boards_editor	.setColour(TextEditor::backgroundColourId, pink_background);
	total_average_moisture_editor	.setColour(TextEditor::backgroundColourId, pink_background);
	total_std_dev_moisture_editor	.setColour(TextEditor::backgroundColourId, pink_background);

	local_number_of_boards_label	.setText("Boards:"	, NotificationType::sendNotification);
	local_average_moisture_label	.setText("Avg:"		, NotificationType::sendNotification);
	local_std_dev_moisture_label	.setText("StDv:"	, NotificationType::sendNotification);
	local_number_of_boards_label	.setJustificationType(Justification::centredRight);
	local_average_moisture_label	.setJustificationType(Justification::centredRight);
	local_std_dev_moisture_label	.setJustificationType(Justification::centredRight);

	local_number_of_boards_editor	.setReadOnly(true);
	local_average_moisture_editor	.setReadOnly(true);
	local_std_dev_moisture_editor	.setReadOnly(true);
	local_number_of_boards_editor	.setJustification(Justification::centredRight);
	local_average_moisture_editor	.setJustification(Justification::centredRight);
	local_std_dev_moisture_editor	.setJustification(Justification::centredRight);
	local_number_of_boards_editor	.setColour(TextEditor::backgroundColourId, blue_background);
	local_average_moisture_editor	.setColour(TextEditor::backgroundColourId, blue_background);
	local_std_dev_moisture_editor	.setColour(TextEditor::backgroundColourId, blue_background);

	// ****************************
	// left of the moisture buckets
	// ****************************

	addAndMakeVisible(run_header);
	addAndMakeVisible(pkg_header);

	run_header.setText("Run:", NotificationType::sendNotification);
	pkg_header.setText("Pkg:", NotificationType::sendNotification);

	// **************************************
	// the buckets with the percentage labels
	// **************************************

	for (size_t idx=0; idx < number_of_buckets; idx++)
	{
		run_moisture_bucket[idx] = 0;
		pkg_moisture_bucket[idx] = 0;

		addAndMakeVisible(total_distribution[idx]);
		addAndMakeVisible(percentages		[idx]);
		addAndMakeVisible(local_distribution[idx]);

		total_distribution[idx].setColour(TextEditor::backgroundColourId, pink_background);
		local_distribution[idx].setColour(TextEditor::backgroundColourId, blue_background);

		total_distribution[idx].setReadOnly(true);
		local_distribution[idx].setReadOnly(true);

		total_distribution[idx].setJustification(Justification::centredRight);
		local_distribution[idx].setJustification(Justification::centredRight);

		percentages[idx].setText(String(percentage_of_first_bucket + idx) + "%", NotificationType::sendNotification);
		percentages[idx].setJustificationType(Justification::centredRight);
	}

	// insert a few fake numbers
	kiln_num_editor	.setText("7");
	run_editor		.setText("026");
	pkg_editor		.setText("118");

	read_all_mc_files();

	// add the chart, and start the timer to redraw it every few seconds
	addAndMakeVisible(chart);
	timerCallback();

	// start the thread that will handle the serial communication
	startThread(10);

	return;
}


TabCanvas::~TabCanvas(void)
{
	signalThreadShouldExit();
	notify();

	stopTimer();

	waitForThreadToExit(5000); // milliseconds

	LOG_MSG("finished destructor for " << (tab_is_active ? "active" : "inactive") << " TabCanvas object with prefix=" << prefix);

	return;
}


void TabCanvas::timerCallback(void)
{
	/* This is called periodically using a timer.  Usually at 1-second intervals, though this can be tweaked via
	 * configuration.
	 *
	 * Note that since we stop the timer, do some redrawing and calculations, and then restart the drawing, it means
	 * the timer will significantly drift over time.  This should not be a concern since all we're doing is updating
	 * portions of the screen, including the chart.
	 */

	stopTimer();

	if (threadShouldExit() == false)
	{
		periodic_update();

		if (timer_frequency_in_milliseconds > 0)
		{
			startTimer(timer_frequency_in_milliseconds); // in milliseconds
		}
	}

	return;
}


void TabCanvas::periodic_update(void)
{
	if (threadShouldExit() == false)
	{
		calculate_averages_and_std_deviations();
	}

	// re-populate the fields with the amounts stored in the buckets
	for (size_t idx=0; idx < number_of_buckets; idx++)
	{
		total_distribution[idx].setText(String(run_moisture_bucket[idx]));
		local_distribution[idx].setText(String(pkg_moisture_bucket[idx]));
	}

	if (threadShouldExit() == false)
	{
		total_number_of_boards_editor.setText(String(run_doubles.size()));
		local_number_of_boards_editor.setText(String(pkg_doubles.size()));
		total_average_moisture_editor.setText(Lox::Numbers::format(run_average, 2));
		local_average_moisture_editor.setText(Lox::Numbers::format(pkg_average, 2));
		total_std_dev_moisture_editor.setText(Lox::Numbers::format(run_std_dev, 2));
		local_std_dev_moisture_editor.setText(Lox::Numbers::format(pkg_std_dev, 2));
	}

//	LOG_MSG("prefix=" << prefix << ": periodic update for RUN: number_of_boards: " << run_doubles.size() << ", average: " << run_average << ", stddev: " << run_std_dev);
//	LOG_MSG("prefix=" << prefix << ": periodic update for PKG: number_of_boards: " << pkg_doubles.size() << ", average: " << pkg_average << ", stddev: " << pkg_std_dev);

	// don't bother redrawing the chart unless we have to
	if (threadShouldExit() == false && (tab_is_active || export_image))
	{
		auto rect = chart.getBounds();
		chart.setImage(draw_chart(rect.getWidth(), rect.getHeight()));
	}

	return;
}


inline double calc_std_dev(const LDouble & ld, const double average)
{
	double std_dev = 0.0;

	// STEP #1:  Find the average of the data set.  This is already done and provided as the variable "average".

	if (ld.size() > 1)
	{
		double sum_of_squares = 0.0;
		for (const double & d : ld)
		{
			// STEP #2:  Take each value in the data set and subtract the mean from it.
			const double val1 = d - average;

			// STEP #3:  Square each of the differences.
			const double val2 = std::pow(val1, 2.0);

			// STEP #4:  Add up all of the results from Step #3 to get the sum of squares.
			sum_of_squares += val2;
		}

		// STEP #5:  Divide the sum of squares (found in step #4) by the number of values in the data set minus one.
		const double val3 = sum_of_squares / static_cast<double>(ld.size() - 1);

		// STEP #6:  Take the square root to get the standard deviation.
		std_dev = std::sqrt(val3);
	}

	return std_dev;
}


void TabCanvas::calculate_averages_and_std_deviations()
{
	// the average is now kept in the class members run_average and pkg_average,
	// so the only thing left to do is calculate the standard deviation

	if (threadShouldExit() == false)
	{
		run_std_dev = calc_std_dev(run_doubles, run_average);
		pkg_std_dev = calc_std_dev(pkg_doubles, pkg_average);
	}

	return;
}


Image TabCanvas::draw_chart(const size_t width, const size_t height)
{
	if (threadShouldExit() || (tab_is_active == false && export_image == false))
	{
		// don't bother creating images for inactive tabs
		return Image();
	}

	Image image(Image::ARGB, width, height, true);
	Graphics g(image);
	g.setColour(Colours::white);
	g.fillRect(0, 0, width, height);

	// draw a border around the entire image
	g.setColour(Colours::black);
	g.drawRect(0, 0, width, height);

	if (width < 75 || height < 75)
	{
		// too small of an image for us to do anything useful
		return image;
	}

	// put the text in the upper-left and upper-right corners
	const int edge_offset = 3;
	g.setColour(Colours::lightgrey);
	g.drawText(cfg().get_str(prefix + "_name"), edge_offset, edge_offset, width - edge_offset * 2, edge_offset * 2, Justification::topLeft);
	g.drawText(Time::getCurrentTime().formatted("%Y-%m-%d %H:%M:%S"), edge_offset, edge_offset, width - edge_offset * 2, edge_offset * 2, Justification::topRight);

	// some numbers are used in many places, so calculate them now so we don't have to do it multiple times
	const float width_float				= static_cast<float>(width);
	const float height_float			= static_cast<float>(height);
	const float h_step					= 1.0f / static_cast<float>(number_of_buckets);
	const float v_step					= 1.0f / 5.0f; // 20%, 40%, 60%, 80%, and 100%
	const float width_of_columns		= h_step * width_float;
	const float one_seventh_of_column	= 1.0f * width_of_columns / 7.0f;
	const float three_seventh_of_column	= 3.0f * width_of_columns / 7.0f;

	/* For drawing purposes, each column is divided into 7 equal parts.
	 * - 1st part is empty space.
	 * - 2nd through 4th is the pink.
	 * - 4th through 6th is the blue.
	 * - 7th is empty space.
	 * Note the 4th section is where the pink and blue parts overlap.
	 */

	// horizontal lines
	g.setColour(Colours::dimgrey);
	for (float y = v_step; y < 1.0; y += v_step)
	{
		g.drawRect(0.0f, y * height_float, width_float, 1.0f);
	}

	// vertical lines
	g.setColour(Colours::dimgrey);
	for (float x = h_step; x < 1.0; x += h_step)
	{
		g.drawRect(x * width_float, 0.0f, 1.0f, height_float);
	}

	// determine the maximum value in all the buckets (initially set to "1" to prevent divide-by-zero later)
	size_t max_run_bucket_value = 1;
	size_t max_pkg_bucket_value = 1;
	for (size_t idx = 0; idx < number_of_buckets; idx++)
	{
		if (run_moisture_bucket[idx] > max_run_bucket_value)	max_run_bucket_value = run_moisture_bucket[idx];
		if (pkg_moisture_bucket[idx] > max_pkg_bucket_value)	max_pkg_bucket_value = pkg_moisture_bucket[idx];
	}

	// draw the columns of data
	for (size_t idx = 0; threadShouldExit() == false && idx < number_of_buckets; idx++)
	{
		const float col_x = idx * h_step * width_float;

		// the graph is based on % of the maximum, not on the raw numbers, so we need to convert this bucket to a percentage
		const float percent_total = (float)run_moisture_bucket[idx] / (float)max_run_bucket_value;
		const float percent_local = (float)pkg_moisture_bucket[idx] / (float)max_pkg_bucket_value;

		// 0% means the bottom of the chart, while 100% means the top of the chart
		// +2 offset means the bar will end 2 pixels from the top of the chart, leaving room for the black border and 1 pixel of white
		const float height_total = (1.0f - percent_total) * height_float + 2.0f;
		const float height_local = (1.0f - percent_local) * height_float + 2.0f;

		// -15 offset means we're going to start 15 pixels from the bottom of the chart, leaving some room for the % text to be printed
		const float bottom_total = height_float - height_total - 15.0f;
		const float bottom_local = height_float - height_local - 15.0f;

		// draw the rectangles in pure white first to wipe away the background grid
		g.setColour(Colours::white.withAlpha(0.85f));
		if (bottom_total > 0.0f)								g.fillRect(col_x + one_seventh_of_column	, height_total, three_seventh_of_column, bottom_total);
		if (bottom_local > 0.0f)								g.fillRect(col_x + three_seventh_of_column	, height_local, three_seventh_of_column, bottom_local);

		// now draw the real semi-transparent rectangles
		if (bottom_total > 0.0f)
		{
			g.setColour(Colours::darkorchid.withAlpha(0.25f));	g.fillRect(col_x + one_seventh_of_column	, height_total, three_seventh_of_column, bottom_total);
			g.setColour(Colours::darkorchid);					g.drawRect(col_x + one_seventh_of_column	, height_total, three_seventh_of_column, bottom_total);
		}

		if (bottom_local > 0.0f)
		{
			g.setColour(Colours::blue.withAlpha(0.25f));		g.fillRect(col_x + three_seventh_of_column	, height_local, three_seventh_of_column, bottom_local);
			g.setColour(Colours::blue);							g.drawRect(col_x + three_seventh_of_column	, height_local, three_seventh_of_column, bottom_local);
		}
	}

	// add % text labels at the bottom of the image
	g.setColour(Colours::black);
	const float label_h = 15;
	const float label_y = height - label_h;
	int percentage	= percentage_of_first_bucket;
	for (float x = 0.0; x < 1.0; x += h_step)
	{
		const float label_x = width_float * x		+ edge_offset;
		const float label_w = width_float * h_step	- edge_offset * 2;
		g.drawText(String(percentage)+"%", label_x, label_y, label_w, label_h, Justification::centredBottom);
		percentage ++;
	}

	// add warnings and errors to the top-left of the image
	if (warnings_and_errors.empty() == false && threadShouldExit() == false)
	{
		g.setColour(Colours::red);
		float y = 0.0f;
		for (const auto & str : warnings_and_errors)
		{
			y += 20.0f;
			g.drawText(str.c_str(), 10.0f, y, width_float - 20.0f, 20.0f, Justification::bottomLeft);
		}
	}

	if (export_image && threadShouldExit() == false)
	{
		// save to a .png file for debug purposes
		File file = File::getSpecialLocation(File::SpecialLocationType::tempDirectory).getChildFile(prefix + "_chart.png");
		file.deleteFile();
		FileOutputStream stream(file);
		PNGImageFormat png;
		png.writeImageToStream(image, stream);
	}

	return image;
}


void TabCanvas::resized(void)
{
	const float height			= 30.0;
	const float margin_size		= 5.0;
	const FlexItem::Margin margin(margin_size);

	FlexBox total_row_flexbox;
	total_row_flexbox.flexDirection		= FlexBox::Direction::row;
	total_row_flexbox.justifyContent	= FlexBox::JustifyContent::spaceAround;
	total_row_flexbox.items.add(FlexItem(kiln_num_label		).withMargin(margin).withWidth(40));
	total_row_flexbox.items.add(FlexItem(kiln_num_editor	).withMargin(margin).withFlex(1.0));
	total_row_flexbox.items.add(FlexItem(run_label			).withMargin(margin).withFlex(1.0));
	total_row_flexbox.items.add(FlexItem(run_editor			).withMargin(margin).withFlex(1.0));
	total_row_flexbox.items.add(FlexItem(pkg_label			).withMargin(margin).withFlex(1.0));
	total_row_flexbox.items.add(FlexItem(pkg_editor			).withMargin(margin).withFlex(1.0));
	// insert a few blank spacers so the row lines up with the moisture distribution row below
	for (size_t idx = 0; idx < number_of_buckets - 11; idx++)
	{
		total_row_flexbox.items.add(FlexItem().withMargin(margin).withFlex(1.0));
	}
	total_row_flexbox.items.add(FlexItem(total_number_of_boards_label	).withMargin(margin).withFlex(1.0));
	total_row_flexbox.items.add(FlexItem(total_number_of_boards_editor	).withMargin(margin).withFlex(1.0));
	total_row_flexbox.items.add(FlexItem(total_average_moisture_label	).withMargin(margin).withFlex(1.0));
	total_row_flexbox.items.add(FlexItem(total_average_moisture_editor	).withMargin(margin).withFlex(1.0));
	total_row_flexbox.items.add(FlexItem(total_std_dev_moisture_label	).withMargin(margin).withFlex(1.0));
	total_row_flexbox.items.add(FlexItem(total_std_dev_moisture_editor	).withMargin(margin).withFlex(1.0));

	FlexBox total_distribution_flexbox;
	total_distribution_flexbox.flexDirection	= FlexBox::Direction::row;
	total_distribution_flexbox.justifyContent	= FlexBox::JustifyContent::spaceAround;
	total_distribution_flexbox.items.add(FlexItem(run_header).withMargin(margin).withWidth(40));
	for (size_t idx = 0; idx < number_of_buckets; idx++)
	{
		total_distribution_flexbox.items.add(FlexItem(total_distribution[idx]).withMargin(margin).withFlex(1.0));
	}

	FlexBox percentage_flexbox;
	percentage_flexbox.flexDirection	= FlexBox::Direction::row;
	percentage_flexbox.justifyContent	= FlexBox::JustifyContent::spaceAround;
	percentage_flexbox.items.add(FlexItem(/* spacer */).withMargin(margin).withWidth(40));	// spacer
	for (size_t idx=0; idx < number_of_buckets; idx++)
	{
		percentage_flexbox.items.add(FlexItem(percentages[idx]).withMargin(margin).withFlex(1.0));
	}

	FlexBox local_distribution_flexbox;
	local_distribution_flexbox.flexDirection	= FlexBox::Direction::row;
	local_distribution_flexbox.justifyContent	= FlexBox::JustifyContent::spaceAround;
	local_distribution_flexbox.items.add(FlexItem(pkg_header).withMargin(margin).withWidth(40));
	for (size_t idx=0; idx < number_of_buckets; idx++)
	{
		local_distribution_flexbox.items.add(FlexItem(local_distribution[idx]).withMargin(margin).withFlex(1.0));
	}

	FlexBox local_row_flexbox;
	local_row_flexbox.flexDirection		= FlexBox::Direction::row;
	local_row_flexbox.justifyContent	= FlexBox::JustifyContent::spaceAround;
	local_row_flexbox.items.add(FlexItem(/* spacer */).withMargin(margin).withWidth(40));
	// insert a few blank spacers so the row lines up with the moisture distribution row below
	for (size_t idx = 0; idx < number_of_buckets - 6; idx++)
	{
		local_row_flexbox.items.add(FlexItem().withMargin(margin).withFlex(1.0));
	}
	local_row_flexbox.items.add(FlexItem(local_number_of_boards_label	).withMargin(margin).withFlex(1.0));
	local_row_flexbox.items.add(FlexItem(local_number_of_boards_editor	).withMargin(margin).withFlex(1.0));
	local_row_flexbox.items.add(FlexItem(local_average_moisture_label	).withMargin(margin).withFlex(1.0));
	local_row_flexbox.items.add(FlexItem(local_average_moisture_editor	).withMargin(margin).withFlex(1.0));
	local_row_flexbox.items.add(FlexItem(local_std_dev_moisture_label	).withMargin(margin).withFlex(1.0));
	local_row_flexbox.items.add(FlexItem(local_std_dev_moisture_editor	).withMargin(margin).withFlex(1.0));

	FlexBox chart_row_flexbox;
	chart_row_flexbox.flexDirection		= FlexBox::Direction::row;
	chart_row_flexbox.justifyContent	= FlexBox::JustifyContent::spaceAround;
	chart_row_flexbox.items.add(FlexItem(/* spacer */).withMargin(margin).withWidth(40));
	chart_row_flexbox.items.add(FlexItem(chart).withFlex(2.0));

	// this is the "main" flexbox for the component
	FlexBox flexbox;
	flexbox.flexDirection									= FlexBox::Direction::column;
	flexbox.items.add(FlexItem(total_row_flexbox			).withHeight(height));
	flexbox.items.add(FlexItem(total_distribution_flexbox	).withHeight(height));
	flexbox.items.add(FlexItem(percentage_flexbox			).withHeight(height));
	flexbox.items.add(FlexItem(local_distribution_flexbox	).withHeight(height));
	flexbox.items.add(FlexItem(local_row_flexbox			).withHeight(height));
	flexbox.items.add(FlexItem(chart_row_flexbox).withFlex(2.0).withMargin(margin));

	auto r = getLocalBounds().toFloat();
	r.reduce(margin_size, margin_size);	// leave a margin around the edge of the window
	flexbox.performLayout( r );

	return;
}


void TabCanvas::run(void)
{
	// this new thread is started in the TabCanvas constructor

	try
	{
		device_thread();
	}
	CATCH_AND_LOG;

	close_serial_device();

	// thread is exiting once we reach here
	return;
}


void TabCanvas::device_thread(void)
{
	// This is started on a secondary thread.

	const std::string device_name	=	cfg().get_str	(prefix + "_device"				);
	const std::string tab_name		=	cfg().get_str	(prefix + "_name"				);
	const int interval_ms			=	cfg().get_int	(prefix + "_interval_ms"		);
	const int read_len				=	cfg().get_int	(prefix + "_read_len"			);
	const bool run_simulation		=	cfg().get_bool	(prefix + "_run_simulation"		);
	const std::regex rx				(	cfg().get_str	(prefix + "_regex_pattern"		));

	LOG_MSG(device_name << ": device thread for " << device_name << " has started (name=\"" << tab_name << "\", prefix=\"" << prefix << "\")");

	for (const auto k : cfg().getAllProperties().getAllKeys())
	{
		if (k.startsWith(prefix))
		{
			const std::string key = k.toStdString();
			LOG_MSG(device_name << ": " << key << "=\"" << cfg().get_str(key) << "\"");
		}
	}

	// The buffer into which bytes are read from the comm port.  The length of this buffer is defined by the *_read_len value in configuration.
	VBytes read_buffer(read_len, '\0');
	LOG_MSG(device_name << ": created a read buffer capable of reading " << read_len << " bytes every " << interval_ms << " milliseconds");

	// give this thread a priority boost
	setPriority(10);

	open_serial_device(device_name);

	auto interval	= std::chrono::milliseconds(interval_ms);
	auto time_now	= std::chrono::high_resolution_clock::now();
	auto next_read	= time_now + interval;

	while (threadShouldExit() == false)
	{
		std::string str;
		if (run_simulation)
		{
			str = get_simulated_string(read_buffer);
		}
		else
		{
			str = read_from_device(read_buffer);
		}

		if (str.empty() == false && threadShouldExit() == false)
		{
			std::smatch matches;
			const bool found = std::regex_search(str, matches, rx);

			if (found)
			{
				// There should be 2 values:  the peak, and the average.  We want to use the average when possible.
				// If the average is <= 1.0, then use the peak instead since sometimes the sensors seem to send the
				// value 0.0 for the average.  See email thread between Dominic and Stephane from 2018-10-18.
				const double peak_value		= (matches.size() > 1 ? std::stod(matches[1]) : 0.0);
				const double average_value	= (matches.size() > 2 ? std::stod(matches[2]) : 0.0);
				double value = average_value;
				if (value < 1.0)
				{
					value = peak_value;
					LOG_MSG(device_name << ": invalid average value from sensor: str=" << String(str).trim().toStdString());
				}
				else
				{
					LOG_MSG(device_name << ": str=" << String(str).trim().toStdString() << ", reading=" << value);
				}

				if (value > 0.0)
				{
					// determine which bucket this should fit into
					int idx = static_cast<int>(value) - percentage_of_first_bucket;
					if (idx < 0)					idx = 0;
					if (idx >= number_of_buckets)	idx = number_of_buckets - 1;

					run_doubles.push_back(value);
					pkg_doubles.push_back(value);

					run_sum += value;
					pkg_sum += value;

					run_average = run_sum / static_cast<double>(run_doubles.size());
					pkg_average = run_sum / static_cast<double>(run_doubles.size());

					run_moisture_bucket[idx] ++;
					pkg_moisture_bucket[idx] ++;
					LOG_MSG(device_name << ": value=" << value << ", incrementing bucket #" << idx << " to " << pkg_moisture_bucket[idx]);

					// every once in a while, write what we have back out to disk so we can restore things when we come back up
					if (0 == pkg_doubles.size() % 100)
					{
						write_mc_file();
					}
				}
			}
			else
			{
				std::string msg;
				for (const char c : str)
				{
					const int i = static_cast<int>(c);
					if (i > 32 && i < 127)	msg += c; // normal ASCII character
					else					msg += "<" + std::to_string(i) + ">"; // non-ASCII printable character
				}
				LOG_MSG(device_name << ": failed to parse string: len=" << str.size() << ": \"" << msg << "\"");
			}
		}

		// now that we've finished reading from the device, sleep until the next scheduled read

		time_now = std::chrono::high_resolution_clock::now();
		if (time_now >= next_read)
		{
			// this is a problem -- either the time interval is too small, or the
			// computer/VM/whatever where we are running is too slow to keep up
			const auto int_ms = std::chrono::duration_cast<std::chrono::milliseconds>(time_now - next_read);

			LOG_MSG(device_name << ": warning: time overflow by " << int_ms.count() << " milliseconds");

			// reset the next read time as if it was now, and hope this doesn't happen again
			next_read = time_now;
		}
		else if (threadShouldExit() == false)
		{
//			LOG_MSG(device_name << ": sleeping for " << std::chrono::duration_cast<std::chrono::milliseconds>(next_read - time_now).count() << " milliseconds");
			std::this_thread::sleep_until(next_read);
		}
		next_read += interval;
	}

	write_mc_file();

	LOG_MSG("device thread for " << device_name << " is exiting (prefix=" << prefix << ")");

	return;
}


void TabCanvas::write_mc_file(void)
{
	// make sure to see the Doxygen documentation to learn about all the fields

	String dirname = cfg().get_str("output_directory");
	dirname = dirname.replaceFirstOccurrenceOf("%Y", Time::getCurrentTime().formatted("%Y"));

	File dir(dirname);
	dir.createDirectory();

	// LINE #1:

	std::ofstream fs(dir.getChildFile(prefix + "_test.mc").getFullPathName().toStdString());
	fs << std::fixed << std::setw(4) << std::setfill('0') << pkg_doubles.size() << ",";

	for (size_t idx = 0; idx < number_of_buckets - 6; idx++)
	{
		fs << std::setw(4) << std::setfill('0') << pkg_moisture_bucket[idx] << ",";
	}

	fs << "000,000,";	// unknown fields

	fs << pkg_std_dev << "," << pkg_average << ",000" << std::endl;

	// LINE #2:

	for (const double d : pkg_doubles)
	{
		fs << std::setw(4) << std::setfill('0') << std::setprecision(1) << d << ",";
	}
	fs << "." << std::endl;

	return;
}


void TabCanvas::read_all_mc_files(void)
{
	String dirname = cfg().get_str("output_directory");
	dirname = dirname.replaceFirstOccurrenceOf("%Y", Time::getCurrentTime().formatted("%Y"));

	File dir(dirname);
	dir.createDirectory();

	const std::string filename = dir.getChildFile(prefix + "_test.mc").getFullPathName().toStdString();

	std::ifstream fs(filename);
	std::string str;
	std::getline(fs, str); // ignore the first line
	std::getline(fs, str); // parse the 2nd line
	fs.close();

//	LOG_MSG("prefix=" << prefix << ": filename=" << filename << ": 2nd line to parse: \"" << str << "\"");

	// replace comma with spaces to make it easier to parse the individual fields
	while (true)
	{
		const size_t pos = str.find(',');
		if (pos == std::string::npos)
		{
			break;
		}
		str[pos] = ' ';
	}

	std::stringstream ss(str);
	while (ss)
	{
		ss >> str;
		if (str.empty() == false && str != ".")
		{
			const double d = std::stod(str);
			if (d > 0.0)
			{
				int idx = static_cast<int>(d) - percentage_of_first_bucket;
				if (idx < 0)					idx = 0;
				if (idx >= number_of_buckets)	idx = number_of_buckets - 1;

				run_moisture_bucket[idx] ++;
				run_doubles.push_back(d);
				run_sum += d;
			}
		}
	}

	if (run_doubles.empty() == false)
	{
		run_average = run_sum / static_cast<double>(run_doubles.size());
	}
	LOG_MSG("prefix=" << prefix << ": filename=" << filename << ": imported " << run_doubles.size() << " existing entries, average is " << run_average);

	return;
}
