GCoderDocument: improve load

Author: u-235

common.pri

@@ -63,6 +63,8 @@ contains(MODULES, GCodeShared) {
                    $$shadowed($${PROJECT_ROOT_PATH}/gcodeshared)
     LIBS += -L$$shadowed($${PROJECT_ROOT_PATH}/gcodeshared)
     LIBS += -lgcodeshared
+    # QTextCodec in Qt6
+    greaterThan(QT_MAJOR_VERSION, 5): QT *= core5compat
 }
 
 contains(MODULES, GCodeWorkShop) {

gcodeshared/gcodeshared.pro

@@ -5,6 +5,8 @@ TARGET = gcodeshared
 CONFIG += staticlib
 
 QT *= widgets serialport network
+# QTextCodec in Qt6
+greaterThan(QT_MAJOR_VERSION, 5): QT += core5compat
 
 include(../common.pri)
 
@@ -38,6 +40,7 @@ HEADERS += include/ui/longjobhelper.h \
     include/utils/medium.h \
     include/utils/configpage.h \
     include/utils/configdialog.h \
+    include/utils/gcode-converter.h \
     include/utils/guessfilename.h \
     include/utils/splitfile.h \
     utils/filepatterns.h
@@ -46,6 +49,7 @@ SOURCES += ui/longjobhelper.cpp \
     utils/medium.cpp \
     utils/configpage.cpp \
     utils/configdialog.cpp \
+    utils/gcode-converter.cpp \
     utils/guessfilename.cpp \
     utils/splitfile.cpp
 

gcodeshared/include/utils/gcode-converter.h

@@ -0,0 +1,58 @@
+/*
+ *  Copyright (C) 2025 Nick Egorrov, nicegorov@yandex.ru
+ *
+ *  This file is part of GCodeWorkShop.
+ *
+ *  GCodeWorkShop is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef UTILS_GCODE_CONVERTER_H
+#define UTILS_GCODE_CONVERTER_H
+
+#include <QByteArray>   // for QByteArray
+
+class QSettings;
+class QString;
+
+
+namespace GCode {
+
+struct Converter {
+	Converter();
+
+	QString fromRawData(const QByteArray& data) const;
+	QByteArray toRawData(const QString& text) const;
+
+	struct Options {
+		QByteArray codecName = QByteArray{};
+		// When loading files, discard "extra" empty lines.
+		bool dropEmptyLine = true;
+		// When loading files, discard the upper 128 code points commonly used for national alphabets.
+		bool dropExtented = false;
+		// When loading files, discard control characters (0x00-0x1f) other than '\n' and '\r'.
+		bool dropControll = true;
+
+		void load(QSettings* settings);
+		void save(QSettings* settings);
+	};
+
+	Options options;
+
+	static Options defaultOptions();
+	static void setDefaultOptions(const Options& options);
+};
+
+} // namespace GCode
+
+#endif // UTILS_GCODE_CONVERTER_H

gcodeshared/utils/gcode-converter.cpp

@@ -0,0 +1,218 @@
+/*
+ *  Copyright (C) 2025 Nick Egorrov, nicegorov@yandex.ru
+ *
+ *  This file is part of GCodeWorkShop.
+ *
+ *  GCodeWorkShop is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <vector>       // for vector
+
+#include <QByteArray>   // for QByteArray
+#include <QSettings>    // for QSettings
+#include <QString>      // for QString
+#include <QVariant>     // for QVariant
+#include <QTextCodec>   // for QTextCodec
+
+#include <utils/gcode-converter.h>  // IWYU pragma: associated
+
+
+namespace GCode {
+namespace Private {
+
+// see https://en.wikipedia.org/wiki/Euclidean_algorithm
+int greatest_common_divisor(int a, int b)
+{
+	while (a != b) {
+		if (a > b) {
+			a -= b;
+		} else {
+			b -= a;
+		}
+	}
+
+	return a;
+}
+
+int greatest_common_divisor(const std::vector<int>& values)
+{
+	if (values.empty()) {
+		return 1;
+	}
+
+	int result = values.front();
+
+	for (int a : values) {
+		result = greatest_common_divisor(a, result);
+	}
+
+	return result;
+}
+
+void add_unique(std::vector<int>& values, int v)
+{
+	for (int i : values) {
+		if (i == v) {
+			return;
+		}
+	}
+
+	values.push_back(v);
+}
+
+QByteArray filterData(const QByteArray& data,
+                      bool dropEmptyLine,
+                      bool dropExtented,
+                      bool dropControll)
+{
+	int newLineCount = 0;
+	int mergedLine = 1;
+
+	if (dropEmptyLine) {
+		std::vector<int> values;
+		newLineCount = 0;
+
+		for (char c : data) {
+			if (c == '\n' || c == '\r') {
+				++newLineCount;
+			} else {
+				if (newLineCount != 0) {
+					add_unique(values, newLineCount);
+					newLineCount = 0;
+				}
+			}
+		}
+
+		mergedLine = greatest_common_divisor(values);
+	}
+
+	QByteArray handled;
+	newLineCount = 0;
+
+	for (char c : data) {
+		if (c == '\n' || c == '\r') {
+			if (++newLineCount > mergedLine) {
+				newLineCount = 1;
+			}
+
+			if (newLineCount > 1) {
+				continue;
+			}
+		} else if (c < 0) {
+			newLineCount = 0;
+
+			if (dropExtented) {
+				continue;
+			}
+		} else if (c < 0x20) {
+			newLineCount = 0;
+
+			if (dropControll) {
+				continue;
+			}
+		} else {
+			newLineCount = 0;
+		}
+
+		handled.push_back(c);
+	}
+
+	return handled;
+}
+
+QString fromRawData(const QByteArray& data,
+                    bool dropEmptyLine,
+                    bool dropExtented,
+                    bool dropControll,
+                    const QByteArray& codec)
+{
+	const QByteArray& handled = filterData(data,
+	                                       dropEmptyLine,
+	                                       dropExtented,
+	                                       dropControll);
+	QTextCodec* textCodec = QTextCodec::codecForName(codec);
+
+	if (textCodec != nullptr) {
+		return textCodec->toUnicode(handled);
+	} else {
+		return QString::fromLocal8Bit(handled);
+	}
+}
+
+QByteArray toRawData(const QString& text, const QByteArray& codec)
+{
+	QTextCodec* textCodec = QTextCodec::codecForName(codec);
+
+	if (textCodec != nullptr) {
+		return textCodec->fromUnicode(text);
+	} else {
+		return text.toLocal8Bit();
+	}
+}
+
+Converter::Options s_defaultOptions{};
+
+} // namespase Private
+} // namespase GCode
+
+
+#define CFG_KEY_CODEC_NAME          "CodecName"
+#define CFG_KEY_DROP_CONTROLL       "DropControll"
+#define CFG_KEY_DROP_EMPTY_LINE     "DropEmptyLine"
+#define CFG_KEY_DROP_EXTENDED       "DropExtented"
+
+void GCode::Converter::Options::load(QSettings* settings)
+{
+	codecName = settings->value(CFG_KEY_CODEC_NAME, codecName).toByteArray();
+	dropControll = settings->value(CFG_KEY_DROP_CONTROLL, dropControll).toBool();
+	dropEmptyLine = settings->value(CFG_KEY_DROP_EMPTY_LINE, dropEmptyLine).toBool();
+	dropExtented = settings->value(CFG_KEY_DROP_EXTENDED, dropExtented).toBool();
+}
+
+void GCode::Converter::Options::save(QSettings* settings)
+{
+	settings->setValue(CFG_KEY_CODEC_NAME, codecName);
+	settings->setValue(CFG_KEY_DROP_CONTROLL, dropControll);
+	settings->setValue(CFG_KEY_DROP_EMPTY_LINE, dropEmptyLine);
+	settings->setValue(CFG_KEY_DROP_EXTENDED, dropExtented);
+}
+
+GCode::Converter::Options GCode::Converter::defaultOptions()
+{
+	return Private::s_defaultOptions;
+}
+
+void GCode::Converter::setDefaultOptions(const Options& options)
+{
+	Private::s_defaultOptions = options;
+}
+
+GCode::Converter::Converter() :
+	options{defaultOptions()}
+{
+}
+
+QString GCode::Converter::fromRawData(const QByteArray& data) const
+{
+	return Private::fromRawData(data,
+	                            options.dropEmptyLine,
+	                            options.dropExtented,
+	                            options.dropControll,
+	                            options.codecName);
+}
+
+QByteArray GCode::Converter::toRawData(const QString& text) const
+{
+	return Private::toRawData(text, options.codecName);
+}

gcodeworkshop/src/gcoderdocument.cpp

@@ -58,6 +58,7 @@ class QMenu;
 #include <documentstyle.h>              // for DocumentStyle, DocumentStyle::Ptr
 #include <documentwidgetproperties.h>   // for DocumentWidgetProperties, DocumentWidgetProperties::Ptr
 #include <gcoderdocument.h>             // IWYU pragma: associated
+#include <utils/gcode-converter.h>      // for Converter
 #include <utils/guessfilename.h>        // for guessFileNameByComments, guessFileNameByProgNum, FileExt
 #include <utils/medium.h>               // for Medium
 #include <utils/removezeros.h>          // for removeZeros
@@ -72,7 +73,6 @@ class QMenu;
 #include "highlightmode.h"          // for MODE_LINUXCNC, MODE_AUTO, MODE_SINUMERIK_840, MODE_FANUC, MODE_HEIDENHAIN
 #include "inlinecalc.h"
 
-
 GCoderDocument::GCoderDocument() : Document(nullptr)
 {
 	m_highlighter = nullptr;
@@ -86,7 +86,6 @@ GCoderDocument::GCoderDocument() : Document(nullptr)
 	m_inLineCalc = new InLineCalc(m_textEdit);
 	connect(m_inLineCalc, SIGNAL(complete(const QString&)), this, SLOT(inLineCalcComplete(const QString&)));
 
-
 	m_textEdit->setWindowIcon(QIcon(":/images/ncfile.png"));
 	m_textEdit->setContextMenuPolicy(Qt::CustomContextMenu);
 	connect(m_textEdit, SIGNAL(customContextMenuRequested(const QPoint&)), this,
@@ -207,12 +206,14 @@ bool GCoderDocument::save()
 
 QByteArray GCoderDocument::rawData() const
 {
-	return text(true).toLocal8Bit();
+	GCode::Converter conv{};
+	return conv.toRawData(text(true));
 }
 
 void GCoderDocument::setRawData(const QByteArray& data)
 {
-	setText(QString::fromLocal8Bit(data));
+	GCode::Converter conv{};
+	setText(conv.fromRawData(data));
 }
 
 QMenu* GCoderDocument::createStandardContextMenu(const QPoint& pos)
@@ -1363,7 +1364,6 @@ void GCoderDocument::highlightCurrentLine()
 				}
 			}
 
-
 			if (m_highlightMode == MODE_SINUMERIK_840) {
 				cursor = textCursor();