Communi: strip IRC formatting, Libtransport: add tests for utf8::replace_invalid and move utf8.h to includes/transport

backends/libcommuni/session.cpp

@@ -23,6 +23,9 @@
 #include <iostream>
 #include <IrcCommand>
 #include <IrcMessage>
+#include <IrcUtil>
+
+#include "backports.h"
 
 #include "ircnetworkplugin.h"
 
@@ -228,13 +231,23 @@ void MyIrcSession::on_messageReceived(IrcMessage *message) {
 	if (m->isAction()) {
 		msg = QString("/me ") + msg;
 	}
+	QString html = "";//msg;
+	CommuniBackport::toPlainText(msg);
+
+	// TODO: Communi produces invalid html now...
+// 	if (html == msg) {
+// 		html = "";
+// 	}
+// 	else {
+// 		html = IrcUtil::messageToHtml(html);
+// 	}
 
 	std::string target = TO_UTF8(m->target().toLower());
 	LOG4CXX_INFO(logger, user << ": Message from " << target);
 	if (target.find("#") == 0) {
 		std::string nickname = TO_UTF8(m->sender().name());
 		correctNickname(nickname);
-		np->handleMessage(user, target + suffix, TO_UTF8(msg), nickname);
+		np->handleMessage(user, target + suffix, TO_UTF8(msg), nickname, TO_UTF8(html));
 	}
 	else {
 		std::string nickname = TO_UTF8(m->sender().name());
@@ -242,7 +255,7 @@ void MyIrcSession::on_messageReceived(IrcMessage *message) {
 		if (m_pms.find(nickname) != m_pms.end()) {
 			if (hasIRCBuddy(m_pms[nickname], nickname)) {
 				LOG4CXX_INFO(logger, nickname);
-				np->handleMessage(user, m_pms[nickname] + suffix, TO_UTF8(msg), nickname, "", "", false, true);
+				np->handleMessage(user, m_pms[nickname] + suffix, TO_UTF8(msg), nickname, TO_UTF8(html), "", false, true);
 				return;
 			}
 			else {
@@ -254,7 +267,7 @@ void MyIrcSession::on_messageReceived(IrcMessage *message) {
 		}
 
 		LOG4CXX_INFO(logger, nickname);
-		np->handleMessage(user, nickname, TO_UTF8(msg));
+		np->handleMessage(user, nickname, TO_UTF8(msg), "", TO_UTF8(html));
 	}
 }
 

include/transport/utf8/checked.h

@@ -94,6 +94,37 @@ namespace utf8
         return result;
     }
 
+    template <typename octet_iterator, typename output_iterator>
+    output_iterator remove_invalid(octet_iterator start, octet_iterator end, output_iterator out)
+    {
+        while (start != end) {
+            octet_iterator sequence_start = start;
+            internal::utf_error err_code = internal::validate_next(start, end);
+            switch (err_code) {
+                case internal::OK :
+                    for (octet_iterator it = sequence_start; it != start; ++it)
+                        *out++ = *it;
+                    break;
+                case internal::NOT_ENOUGH_ROOM:
+                    throw not_enough_room();
+                case internal::INVALID_LEAD:
+//                     append (replacement, out);
+                    ++start;
+                    break;
+                case internal::INCOMPLETE_SEQUENCE:
+                case internal::OVERLONG_SEQUENCE:
+                case internal::INVALID_CODE_POINT:
+//                     append (replacement, out);
+                    ++start;
+                    // just one replacement mark for the sequence
+                    while (internal::is_trail(*start) && start != end)
+                        ++start;
+                    break;
+            }
+        }
+        return out;
+    }
+
     template <typename octet_iterator, typename output_iterator>
     output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
     {

src/networkpluginserver.cpp

@@ -52,7 +52,7 @@
 #include "boost/date_time/posix_time/posix_time.hpp"
 #include "boost/signal.hpp"
 
-#include "utf8.h"
+#include "transport/utf8.h"
 
 #include <Swiften/FileTransfer/ReadBytestream.h>
 #include <Swiften/Elements/StreamInitiationFileInfo.h>
@@ -246,8 +246,8 @@ static void handleBuddyPayload(LocalBuddy *buddy, const pbnetwork::Buddy &payloa
 	// Change groups if it's not empty. The same as above...
 	std::vector<std::string> groups;
 	for (int i = 0; i < payload.group_size(); i++) {
-		std::string group = payload.group(i);
-		utf8::replace_invalid(payload.group(i).begin(), payload.group(i).end(), group.begin(), '_');
+		std::string group;
+		utf8::replace_invalid(payload.group(i).begin(), payload.group(i).end(), std::back_inserter(group), '_');
 		groups.push_back(group);
 	}
 	if (!groups.empty()) {
@@ -490,16 +490,16 @@ void NetworkPluginServer::handleVCardPayload(const std::string &data) {
 		// TODO: ERROR
 		return;
 	}
-	std::string field = payload.fullname();
+	std::string field;
 
 	boost::shared_ptr<Swift::VCard> vcard(new Swift::VCard());
 
-	utf8::replace_invalid(payload.fullname().begin(), payload.fullname().end(), field.begin(), '_');
+	utf8::replace_invalid(payload.fullname().begin(), payload.fullname().end(), std::back_inserter(field), '_');
 	vcard->setFullName(field);
 
-	field = payload.nickname();
+	field.clear();
 
-	utf8::replace_invalid(payload.nickname().begin(), payload.nickname().end(), field.begin(), '_');
+	utf8::replace_invalid(payload.nickname().begin(), payload.nickname().end(), std::back_inserter(field), '_');
 	vcard->setNickname(field);
 
 	vcard->setPhoto(Swift::createByteArray(payload.photo()));

src/tests/util.cpp

@@ -18,6 +18,7 @@
 #include "Swiften/Server/ServerFromClientSession.h"
 #include "Swiften/Parser/PayloadParsers/FullPayloadParserFactoryCollection.h"
 #include "basictest.h"
+#include "transport/utf8.h"
 
 #include "transport/util.h"
 
@@ -27,6 +28,7 @@ class UtilTest : public CPPUNIT_NS :: TestFixture{
 	CPPUNIT_TEST_SUITE(UtilTest);
 	CPPUNIT_TEST(encryptDecryptPassword);
 	CPPUNIT_TEST(serializeGroups);
+	CPPUNIT_TEST(replaceInvalid);
 	CPPUNIT_TEST_SUITE_END();
 
 	public:
@@ -63,6 +65,18 @@ class UtilTest : public CPPUNIT_NS :: TestFixture{
 		CPPUNIT_ASSERT_EQUAL(std::string("Buddies2"), StorageBackend::deserializeGroups(g)[1]);
 	}
 
+	void replaceInvalid() {
+		std::string x("test\x80\xe0\xa0\xc0\xaf\xed\xa0\x80test");
+		std::string a;
+		CPPUNIT_ASSERT(x.end() != utf8::find_invalid(x.begin(), x.end()));
+		utf8::replace_invalid(x.begin(), x.end(), std::back_inserter(a), '_');
+		CPPUNIT_ASSERT_EQUAL(std::string("test____test"), a);
+
+		a = "";
+		utf8::remove_invalid(x.begin(), x.end(), std::back_inserter(a));
+		CPPUNIT_ASSERT_EQUAL(std::string("testtest"), a);
+	}
+
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION (UtilTest);