fix tokenize

base/str.cc:
    Fix tokenize so that it doesn't behave incorrectly when there
    are empty strings.
test/tokentest.cc:
    Clean up the test function so it's easier to see what's going on

--HG--
extra : convert_revision : c7a3db7bc516d3575b1cc4ab7afbd0f1fbe1ec6f
This commit is contained in:
Nathan Binkert 2005-06-22 09:52:02 -04:00
parent 11894d3b4b
commit c95e1281fc
2 changed files with 50 additions and 37 deletions

View file

@ -26,11 +26,10 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <iostream>
#include <string.h>
#include <ctype.h>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
@ -75,15 +74,17 @@ tokenize(vector<string>& v, const string &s, char token, bool ignore)
string::size_type first = 0;
string::size_type last = s.find_first_of(token);
if (ignore) {
if (last == first) {
while (last == first)
last = s.find_first_of(token, ++first);
if (s.empty())
return;
if (last == string::npos) {
v.push_back(s);
return;
}
if (ignore && last == first) {
while (last == first)
last = s.find_first_of(token, ++first);
if (last == string::npos) {
if (first != s.size())
v.push_back(s.substr(first));
return;
}
}

View file

@ -26,7 +26,7 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <iostream.h>
#include <iostream>
#include <string>
#include <vector>
@ -35,35 +35,47 @@
int
main(int argc, char *argv[])
{
if (argc != 3) {
cout << "Usage: " << argv[0] << " <string> <token>\n";
exit(1);
}
using namespace std;
int i;
string test = argv[1];
vector<string> tokens1;
vector<string> tokens2;
char token = argv[2][0];
if (argc != 3) {
cout << "Usage: " << argv[0] << " <string> <token>\n";
exit(1);
}
cout << "string = \"" << test << "\", token = \'" << token << "\'\n";
cout << "testing without ignore\n";
tokenize(tokens1, test, token, false);
int i;
string test = argv[1];
vector<string> tokens1;
vector<string> tokens2;
char token = argv[2][0];
if (tokens1.size()) {
for (i = 0; i < tokens1.size() - 1; i++)
cout << tokens1[i] << "(" << tokens1[i].size() << "), ";
cout << tokens1[i] << "(" << tokens1[i].size() << ")\n";
}
cout << "string = \"" << test << "\", token = \'" << token << "\'\n";
cout << "testing without ignore\n";
tokenize(tokens1, test, token, false);
cout << "testing with ignore\n";
tokenize(tokens2, test, token, true);
if (tokens1.size()) {
int size = tokens1.size();
cout << "size = " << size << "\n";
for (i = 0; i < size; i++) {
cout << "'" << tokens1[i] << "' (" << tokens1[i].size()
<< ")" << ((i == size - 1) ? "\n" : ", ");
}
} else {
cout << "no tokens" << endl;
}
if (tokens2.size()) {
for (i = 0; i < tokens2.size() - 1; i++)
cout << tokens2[i] << "(" << tokens2[i].size() << "), ";
cout << tokens2[i] << "(" << tokens2[i].size() << ")\n";
}
cout << "testing with ignore\n";
tokenize(tokens2, test, token, true);
return 0;
if (tokens2.size()) {
int size = tokens2.size();
cout << "size = " << size << "\n";
for (i = 0; i < size; i++) {
cout << "'" << tokens2[i] << "' (" << tokens2[i].size()
<< ")" << ((i == size - 1) ? "\n" : ", ");
}
} else {
cout << "no tokens" << endl;
}
return 0;
}